
"""simple docstring"""




from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


A__      :	Dict          =    logging.get_logger(__name__)

A__      :	Any          =    {
    "t5-small": "https://huggingface.co/t5-small/resolve/main/config.json",
    "t5-base": "https://huggingface.co/t5-base/resolve/main/config.json",
    "t5-large": "https://huggingface.co/t5-large/resolve/main/config.json",
    "t5-3b": "https://huggingface.co/t5-3b/resolve/main/config.json",
    "t5-11b": "https://huggingface.co/t5-11b/resolve/main/config.json",
}







class   _lowercase						(       _UpperCAmelCase				):


     '''simple docstring'''





     _A								=							't5'
     _A								=							['past_key_values']
     _A								=							{'hidden_size': 'd_model', 'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}



     def __init__(      self ,			__UpperCamelCase=3_21_28 ,			__UpperCamelCase=5_12 ,			__UpperCamelCase=64 ,			__UpperCamelCase=20_48 ,			__UpperCamelCase=6 ,			__UpperCamelCase=None ,			__UpperCamelCase=8 ,			__UpperCamelCase=32 ,			__UpperCamelCase=1_28 ,			__UpperCamelCase=0.1 ,			__UpperCamelCase=1E-6 ,			__UpperCamelCase=1.0 ,			__UpperCamelCase="relu" ,			__UpperCamelCase=True ,			__UpperCamelCase=True ,			__UpperCamelCase=0 ,			__UpperCamelCase=1 ,			**__UpperCamelCase ,			)->					Optional[Any]:
          UpperCAmelCase__							: Dict 							=						vocab_size
          UpperCAmelCase__							: Dict 							=						d_model
          UpperCAmelCase__							: List[str] 							=						d_kv
          UpperCAmelCase__							: Tuple 							=						d_ff
          UpperCAmelCase__							: str 							=						num_layers
          UpperCAmelCase__							: List[Any] 							=						(
              num_decoder_layers if num_decoder_layers is not None else self.num_layers
          )  # default = symmetry
          UpperCAmelCase__							: Union[str, Any] 							=						num_heads
          UpperCAmelCase__							: Any 							=						relative_attention_num_buckets
          UpperCAmelCase__							: Optional[Any] 							=						relative_attention_max_distance
          UpperCAmelCase__							: str 							=						dropout_rate
          UpperCAmelCase__							: Optional[Any] 							=						layer_norm_epsilon
          UpperCAmelCase__							: Tuple 							=						initializer_factor
          UpperCAmelCase__							: Optional[Any] 							=						feed_forward_proj
          UpperCAmelCase__							: str 							=						use_cache

          UpperCAmelCase__							: Any 							=						self.feed_forward_proj.split("-"  )
          UpperCAmelCase__							: List[str] 							=						act_info[-1]
          UpperCAmelCase__							: Optional[Any] 							=						act_info[0] == '''gated'''

          if len(lowerCamelCase_  ) > 1 and act_info[0] != "gated" or len(lowerCamelCase_  ) > 2:
               raise ValueError(
                   F"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
                   "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
                   "\'gated-gelu\' or \'relu\'"  )

          # for backwards compatibility
          if feed_forward_proj == "gated-gelu":
               UpperCAmelCase__							: Dict 							=						'''gelu_new'''

          super().__init__(
              pad_token_id=lowerCamelCase_ ,			eos_token_id=lowerCamelCase_ ,			is_encoder_decoder=lowerCamelCase_ ,			**lowerCamelCase_ ,			)







class   _lowercase						(       _UpperCAmelCase				):


     '''simple docstring'''





     @property
     def  lowerCAmelCase__       (      self  )->					Mapping[str, Mapping[int, str]]:
          UpperCAmelCase__							: Tuple 							=						{
              '''input_ids''': {0: '''batch''', 1: '''encoder_sequence'''},
              '''attention_mask''': {0: '''batch''', 1: '''encoder_sequence'''},
          }
          if self.use_past:
               UpperCAmelCase__							: List[Any] 							=						'''past_encoder_sequence + sequence'''
               UpperCAmelCase__							: List[Any] 							=						{0: '''batch'''}
               UpperCAmelCase__							: Dict 							=						{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
          else:
               UpperCAmelCase__							: Optional[int] 							=						{0: '''batch''', 1: '''decoder_sequence'''}
               UpperCAmelCase__							: Union[str, Any] 							=						{0: '''batch''', 1: '''decoder_sequence'''}

          if self.use_past:
               self.fill_with_past_key_values_(lowerCamelCase_ ,			direction="inputs"  )

          return common_inputs



     @property
     def  lowerCAmelCase__       (      self  )->					int:
          return 13





"""simple docstring"""




import warnings
from pathlib import Path
from typing import List, Tuple, Union

import fire
from torch import nn

from transformers import AutoModelForSeqaSeqLM, AutoTokenizer, PreTrainedModel
from transformers.utils import logging


A__      :	Tuple          =    logging.get_logger(__name__)



def 							a__ (				lowerCAmelCase	:       nn.ModuleList       ,    lowerCAmelCase	:       nn.ModuleList       ,    lowerCAmelCase	:       List[int]		):






     '''simple docstring'''
     UpperCAmelCase__							: Optional[Any] 							=						nn.ModuleList([src_layers[i] for i in layers_to_copy]		)
     assert len(lowerCAmelCase		) == len(lowerCAmelCase		), F"{len(lowerCAmelCase		)} != {len(lowerCAmelCase		)}"
     dest_layers.load_state_dict(layers_to_copy.state_dict()		)


A__      :	List[Any]          =    {
    # maps  num layers in teacher -> num_layers in student -> which teacher layers to copy.
    # 12: bart, 16: pegasus, 6: marian/Helsinki-NLP
    12: {
        1: [0],  # This says that if the teacher has 12 layers and the student has 1, copy layer 0 of the teacher
        2: [0, 6],
        3: [0, 6, 11],
        4: [0, 4, 8, 11],
        6: [0, 2, 4, 7, 9, 11],
        9: [0, 1, 2, 4, 5, 7, 9, 10, 11],
        12: list(range(12)),
    },
    16: {  # maps  num layers in student -> which teacher layers to copy
        1: [0],
        2: [0, 15],
        3: [0, 8, 15],
        4: [0, 5, 10, 15],
        6: [0, 3, 6, 9, 12, 15],
        8: [0, 2, 4, 6, 8, 10, 12, 15],
        9: [0, 1, 3, 5, 7, 9, 11, 13, 15],
        12: [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 13, 15],
        16: list(range(16)),
    },
    6: {1: [0], 2: [0, 5], 3: [0, 2, 5], 4: [0, 1, 3, 5], 6: list(range(6))},
}
A__      :	List[Any]          =    {
    # maps  num layers in student -> which teacher layers to copy.
    6: {1: [5], 2: [3, 5], 3: [1, 4, 5], 4: [1, 2, 4, 5]},
    12: {1: [11], 2: [5, 11], 3: [3, 7, 11], 6: [1, 3, 5, 8, 10, 11]},
    16: {1: [15], 4: [4, 9, 12, 15], 8: [1, 3, 5, 7, 9, 11, 13, 15]},
}



def 							a__ (				lowerCAmelCase	:       Dict       ,    lowerCAmelCase	:       Dict		):






     '''simple docstring'''
     try:
          UpperCAmelCase__							: Tuple 							=						LAYERS_TO_COPY[n_teacher][n_student]
          return val
     except KeyError:
          if n_student != n_teacher:
               warnings.warn(
                   F"no hardcoded layers to copy for teacher {n_teacher} -> student {n_student}, defaulting to first"
                   F" {n_student}"		)
          return list(range(lowerCAmelCase		)		)



def 							a__ (				lowerCAmelCase	:       int       ,    lowerCAmelCase	:       Tuple		):






     '''simple docstring'''
     if n_student > n_teacher:
          raise ValueError(F"Cannot perform intermediate supervision for student {n_student} > teacher {n_teacher}"		)
     elif n_teacher == n_student:
          return list(range(lowerCAmelCase		)		)
     elif n_student == 1:
          return [n_teacher - 1]
     else:
          return LAYERS_TO_SUPERVISE[n_teacher][n_student]



def 							a__ (				lowerCAmelCase	:       Union[str, PreTrainedModel]       ,    lowerCAmelCase	:       Union[str, Path] = "student"       ,    lowerCAmelCase	:       Union[int, None] = None       ,    lowerCAmelCase	:       Union[int, None] = None       ,    lowerCAmelCase	:       List[str]=False       ,    lowerCAmelCase	:       List[str]=None       ,    lowerCAmelCase	:       List[str]=None       ,    **lowerCAmelCase	:       List[str]       ,    ):






     '''simple docstring'''
     UpperCAmelCase__							: List[str] 							=						"encoder_layers and decoder_layers cannot be both None-- you would just have an identical teacher."
     assert (e is not None) or (d is not None), _msg
     if isinstance(lowerCAmelCase       ,    lowerCAmelCase		):
          AutoTokenizer.from_pretrained(lowerCAmelCase		).save_pretrained(lowerCAmelCase		)  # purely for convenience
          UpperCAmelCase__							: List[str] 							=						AutoModelForSeqaSeqLM.from_pretrained(lowerCAmelCase		).eval()
     else:
          assert isinstance(lowerCAmelCase       ,    lowerCAmelCase		), F"teacher must be a model or string got type {type(lowerCAmelCase		)}"
     UpperCAmelCase__							: int 							=						teacher.config.to_diff_dict()

     try:
          UpperCAmelCase__							,	UpperCAmelCase__							: Optional[Any] 							=						teacher.config.encoder_layers, teacher.config.decoder_layers
          if e is None:
               UpperCAmelCase__							: Tuple 							=						teacher_e
          if d is None:
               UpperCAmelCase__							: str 							=						teacher_d
          init_kwargs.update({"encoder_layers": e, "decoder_layers": d}		)
     except AttributeError:  # T5
          if hasattr(teacher.config       ,    "num_encoder_layers"		):
               UpperCAmelCase__							,	UpperCAmelCase__							: Union[str, Any] 							=						teacher.config.num_encoder_layers, teacher.config.num_decoder_layers
          else:
               UpperCAmelCase__							,	UpperCAmelCase__							: Union[str, Any] 							=						teacher.config.num_layers, teacher.config.num_decoder_layers
          if e is None:
               UpperCAmelCase__							: Optional[Any] 							=						teacher_e
          if d is None:
               UpperCAmelCase__							: Optional[Any] 							=						teacher_d
          if hasattr(teacher.config       ,    "num_encoder_layers"		):
               init_kwargs.update({"num_encoder_layers": e, "num_decoder_layers": d}		)
          else:
               init_kwargs.update({"num_layers": e, "num_decoder_layers": d}		)

    # Kwargs to instantiate student: teacher kwargs with updated layer numbers + **extra_config_kwargs
     init_kwargs.update(lowerCAmelCase		)

     # Copy weights
     UpperCAmelCase__							: Tuple 							=						teacher.config_class(**lowerCAmelCase		)
     UpperCAmelCase__							: List[str] 							=						AutoModelForSeqaSeqLM.from_config(lowerCAmelCase		)
     # Start by copying the full teacher state dict this will copy the first N teacher layers to the student.
     UpperCAmelCase__							: Optional[int] 							=						student.load_state_dict(teacher.state_dict()       ,    strict=lowerCAmelCase		)
     assert info.missing_keys == [], info.missing_keys  # every student key should have a teacher keys.

     if copy_first_teacher_layers:  # Our copying is done. We just log and save
          UpperCAmelCase__							,	UpperCAmelCase__							: int 							=						list(range(lowerCAmelCase		)		), list(range(lowerCAmelCase		)		)
          logger.info(
              F"Copied encoder layers {e_layers_to_copy} and decoder layers {d_layers_to_copy}. Saving them to"
              F" {save_path}"		)
          student.save_pretrained(lowerCAmelCase		)
          return student, e_layers_to_copy, d_layers_to_copy

     # Decide which layers of the teacher to copy. Not exactly alternating -- we try to keep first and last layer.
     if e_layers_to_copy is None:
          UpperCAmelCase__							: List[int] 							=						pick_layers_to_copy(lowerCAmelCase       ,    lowerCAmelCase		)
     if d_layers_to_copy is None:
          UpperCAmelCase__							: List[int] 							=						pick_layers_to_copy(lowerCAmelCase       ,    lowerCAmelCase		)

     try:
          if hasattr(
              lowerCAmelCase       ,    "prophetnet"		):  # For ProphetNet, student.model.encoder.layers is called student.prophetnet.encoder.layers
               copy_layers(teacher.prophetnet.encoder.layers       ,    student.prophetnet.encoder.layers       ,    lowerCAmelCase		)
               copy_layers(teacher.prophetnet.decoder.layers       ,    student.prophetnet.decoder.layers       ,    lowerCAmelCase		)
          else:
               copy_layers(teacher.model.encoder.layers       ,    student.model.encoder.layers       ,    lowerCAmelCase		)
               copy_layers(teacher.model.decoder.layers       ,    student.model.decoder.layers       ,    lowerCAmelCase		)
     except AttributeError:  # For t5, student.model.encoder.layers is called student.encoder.block
          copy_layers(teacher.encoder.block       ,    student.encoder.block       ,    lowerCAmelCase		)
          copy_layers(teacher.decoder.block       ,    student.decoder.block       ,    lowerCAmelCase		)
     logger.info(
         F"Copied encoder layers {e_layers_to_copy} and decoder layers {d_layers_to_copy}. Saving them to {save_path}"		)
     UpperCAmelCase__							: int 							=						{
         "teacher_type": teacher.config.model_type,
         "copied_encoder_layers": e_layers_to_copy,
         "copied_decoder_layers": d_layers_to_copy,
     }
     student.save_pretrained(lowerCAmelCase		)
     # Save information about copying for easier reproducibility

     return student, e_layers_to_copy, d_layers_to_copy


if __name__ == "__main__":
    fire.Fire(create_student_by_copying_alternating_layers)



