


'''simple docstring'''







import argparse

import torch

from transformers import BertForMaskedLM


if __name__ == "__main__":
       lowerCAmelCase    :     Any					=						argparse.ArgumentParser(
           description=(
               """Extraction some layers of the full BertForMaskedLM or RObertaForMaskedLM for Transfer Learned"""
               """ Distillation"""
           )
       )
       parser.add_argument("""--model_type""", default="""bert""", choices=["""bert"""])
       parser.add_argument("""--model_name""", default="""bert-base-uncased""", type=str)
       parser.add_argument("""--dump_checkpoint""", default="""serialization_dir/tf_bert-base-uncased_0247911.pth""", type=str)
       parser.add_argument("""--vocab_transform""", action="""store_true""")
       lowerCAmelCase    :     Any					=						parser.parse_args()

       if args.model_type == "bert":
              lowerCAmelCase    :     Tuple					=						BertForMaskedLM.from_pretrained(args.model_name)
              lowerCAmelCase    :     List[Any]					=						"""bert"""
       else:
              raise ValueError("""args.model_type should be \"bert\".""")

       lowerCAmelCase    :     List[Any]					=						model.state_dict()
       lowerCAmelCase    :     Optional[int]					=						{}

       for w in ["word_embeddings", "position_embeddings"]:
              lowerCAmelCase    :     Union[str, Any]					=						state_dict[F'''{prefix}.embeddings.{w}.weight''']
       for w in ["weight", "bias"]:
              lowerCAmelCase    :     Dict					=						state_dict[F'''{prefix}.embeddings.LayerNorm.{w}''']

       lowerCAmelCase    :     int					=						0
       for teacher_idx in [0, 2, 4, 7, 9, 11]:
              for w in ["weight", "bias"]:
                     lowerCAmelCase    :     str					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.query.{w}'''
                     ]
                     lowerCAmelCase    :     List[str]					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.key.{w}'''
                     ]
                     lowerCAmelCase    :     Optional[Any]					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.attention.self.value.{w}'''
                     ]

                     lowerCAmelCase    :     str					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.dense.{w}'''
                     ]
                     lowerCAmelCase    :     Optional[int]					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.attention.output.LayerNorm.{w}'''
                     ]

                     lowerCAmelCase    :     str					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.intermediate.dense.{w}'''
                     ]
                     lowerCAmelCase    :     Tuple					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.output.dense.{w}'''
                     ]
                     lowerCAmelCase    :     List[str]					=						state_dict[
                         F'''{prefix}.encoder.layer.{teacher_idx}.output.LayerNorm.{w}'''
                     ]
              std_idx += 1

       lowerCAmelCase    :     str					=						state_dict["""cls.predictions.decoder.weight"""]
       lowerCAmelCase    :     Dict					=						state_dict["""cls.predictions.bias"""]
       if args.vocab_transform:
              for w in ["weight", "bias"]:
                     lowerCAmelCase    :     List[str]					=						state_dict[F'''cls.predictions.transform.dense.{w}''']
                     lowerCAmelCase    :     List[Any]					=						state_dict[F'''cls.predictions.transform.LayerNorm.{w}''']

       print(F'''N layers selected for distillation: {std_idx}''')
       print(F'''Number of params transferred for distillation: {len(compressed_sd.keys())}''')

       print(F'''Save transferred checkpoint to {args.dump_checkpoint}.''')
       torch.save(compressed_sd, args.dump_checkpoint)






'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase    :     Any					=						{
    """configuration_x_clip""": [
        """XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """XCLIPConfig""",
        """XCLIPTextConfig""",
        """XCLIPVisionConfig""",
    ],
    """processing_x_clip""": ["""XCLIPProcessor"""],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCAmelCase    :     Optional[Any]					=						[
           """XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """XCLIPModel""",
           """XCLIPPreTrainedModel""",
           """XCLIPTextModel""",
           """XCLIPVisionModel""",
       ]

if TYPE_CHECKING:
       from .configuration_x_clip import (
           XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           XCLIPConfig,
           XCLIPTextConfig,
           XCLIPVisionConfig,
       )
       from .processing_x_clip import XCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_x_clip import (
                  XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XCLIPModel,
                  XCLIPPreTrainedModel,
                  XCLIPTextModel,
                  XCLIPVisionModel,
              )

else:
       import sys

       lowerCAmelCase    :     Dict					=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

