

"""simple docstring"""




import argparse
import os
import re

import tensorflow as tf
import torch

from transformers import BertConfig, BertModel
from transformers.utils import logging


logging.set_verbosity_info()
A_							:			Optional[Any]											= logging.get_logger(__name__)

def 	A       (     snake_case__		,				snake_case__		,				snake_case__    ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__								=      os.path.abspath(snake_case__    )
			logger.info(f"""Converting TensorFlow checkpoint from {tf_path}"""    )
			# Load weights from TF model
			SCREAMING_SNAKE_CASE__								=      tf.train.list_variables(snake_case__    )
			SCREAMING_SNAKE_CASE__								=      []
			SCREAMING_SNAKE_CASE__								=      []
			SCREAMING_SNAKE_CASE__								=      []
			for full_name, shape in init_vars:
						# logger.info(f"Loading TF weight {name} with shape {shape}")
						SCREAMING_SNAKE_CASE__								=      full_name.split("""/"""    )
						if full_name == "_CHECKPOINTABLE_OBJECT_GRAPH" or name[0] in ["global_step", "save_counter"]:
									logger.info(f"""Skipping non-model layer {full_name}"""    )
									continue
						if "optimizer" in full_name:
									logger.info(f"""Skipping optimization layer {full_name}"""    )
									continue
						if name[0] == "model":
									# ignore initial 'model'
									SCREAMING_SNAKE_CASE__								=      name[1:]
						# figure out how many levels deep the name is
						SCREAMING_SNAKE_CASE__								=      0
						for _name in name:
									if _name.startswith("""layer_with_weights"""    ):
												depth += 1
									else:
												break
						layer_depth.append(snake_case__    )
						# read data
						SCREAMING_SNAKE_CASE__								=      tf.train.load_variable(snake_case__		,				snake_case__    )
						names.append("""/""".join(snake_case__    )    )
						arrays.append(snake_case__    )
			logger.info(f"""Read a total of {len(snake_case__    ):,} layers"""    )

			# Sanity check
			if len(set(snake_case__    )    ) != 1:
						raise ValueError(f"""Found layer names with different depths (layer depth {list(set(snake_case__    )    )})"""    )
			SCREAMING_SNAKE_CASE__								=      list(set(snake_case__    )    )[0]
			if layer_depth != 1:
						raise ValueError(
						    """The model contains more than just the embedding/encoder layers. This script does not handle MLM/NSP"""
						    """ heads."""    )

			# convert layers
			logger.info("""Converting weights..."""    )
			for full_name, array in zip(snake_case__		,				snake_case__    ):
						SCREAMING_SNAKE_CASE__								=      full_name.split("""/"""    )
						SCREAMING_SNAKE_CASE__								=      model
						SCREAMING_SNAKE_CASE__								=      []
						for i, m_name in enumerate(snake_case__    ):
									if m_name == ".ATTRIBUTES":
												# variable names end with .ATTRIBUTES/VARIABLE_VALUE
												break
									if m_name.startswith("""layer_with_weights"""    ):
												SCREAMING_SNAKE_CASE__								=      int(m_name.split("""-"""    )[-1]    )
												if layer_num <= 2:
															# embedding layers
															# layer_num 0: word_embeddings
															# layer_num 1: position_embeddings
															# layer_num 2: token_type_embeddings
															continue
												elif layer_num == 3:
															# embedding LayerNorm
															trace.extend(["""embeddings""", """LayerNorm"""]    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""embeddings"""    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""LayerNorm"""    )
												elif layer_num > 3 and layer_num < config.num_hidden_layers + 4:
															# encoder layers
															trace.extend(["""encoder""", """layer""", str(layer_num - 4    )]    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""encoder"""    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""layer"""    )
															SCREAMING_SNAKE_CASE__								=      pointer[layer_num - 4]
												elif layer_num == config.num_hidden_layers + 4:
															# pooler layer
															trace.extend(["""pooler""", """dense"""]    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""pooler"""    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""dense"""    )
									elif m_name == "embeddings":
												trace.append("""embeddings"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""embeddings"""    )
												if layer_num == 0:
															trace.append("""word_embeddings"""    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""word_embeddings"""    )
												elif layer_num == 1:
															trace.append("""position_embeddings"""    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""position_embeddings"""    )
												elif layer_num == 2:
															trace.append("""token_type_embeddings"""    )
															SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""token_type_embeddings"""    )
												else:
															raise ValueError(f"""Unknown embedding layer with name {full_name}"""    )
												trace.append("""weight"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""weight"""    )
									elif m_name == "_attention_layer":
												# self-attention layer
												trace.extend(["""attention""", """self"""]    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""attention"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""self"""    )
									elif m_name == "_attention_layer_norm":
												# output attention norm
												trace.extend(["""attention""", """output""", """LayerNorm"""]    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""attention"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""output"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""LayerNorm"""    )
									elif m_name == "_attention_output_dense":
												# output attention dense
												trace.extend(["""attention""", """output""", """dense"""]    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""attention"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""output"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""dense"""    )
									elif m_name == "_output_dense":
												# output dense
												trace.extend(["""output""", """dense"""]    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""output"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""dense"""    )
									elif m_name == "_output_layer_norm":
												# output dense
												trace.extend(["""output""", """LayerNorm"""]    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""output"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""LayerNorm"""    )
									elif m_name == "_key_dense":
												# attention key
												trace.append("""key"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""key"""    )
									elif m_name == "_query_dense":
												# attention query
												trace.append("""query"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""query"""    )
									elif m_name == "_value_dense":
												# attention value
												trace.append("""value"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""value"""    )
									elif m_name == "_intermediate_dense":
												# attention intermediate dense
												trace.extend(["""intermediate""", """dense"""]    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""intermediate"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""dense"""    )
									elif m_name == "_output_layer_norm":
												# output layer norm
												trace.append("""output"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""output"""    )
									# weights & biases
									elif m_name in ["bias", "beta"]:
												trace.append("""bias"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""bias"""    )
									elif m_name in ["kernel", "gamma"]:
												trace.append("""weight"""    )
												SCREAMING_SNAKE_CASE__								=      getattr(snake_case__		,				"""weight"""    )
									else:
												logger.warning(f"""Ignored {m_name}"""    )
        # for certain layers reshape is necessary
						SCREAMING_SNAKE_CASE__								=      """.""".join(snake_case__    )
						if re.match(R"""(\S+)\.attention\.self\.(key|value|query)\.(bias|weight)"""		,				snake_case__    ) or re.match(
						    R"""(\S+)\.attention\.output\.dense\.weight"""		,				snake_case__    ):
									SCREAMING_SNAKE_CASE__								=      array.reshape(pointer.data.shape    )
						if "kernel" in full_name:
									SCREAMING_SNAKE_CASE__								=      array.transpose()
						if pointer.shape == array.shape:
									SCREAMING_SNAKE_CASE__								=      torch.from_numpy(snake_case__    )
						else:
									raise ValueError(
									    f"""Shape mismatch in layer {full_name}: Model expects shape {pointer.shape} but layer contains shape:"""
									    f""" {array.shape}"""    )
						logger.info(f"""Successfully set variable {full_name} to PyTorch layer {trace}"""    )
			return model

def 	A       (     snake_case__		,				snake_case__		,				snake_case__    ):




			'''simple docstring'''


			logger.info(f"""Loading model based on config from {config_path}..."""    )
			SCREAMING_SNAKE_CASE__								=      BertConfig.from_json_file(snake_case__    )
			SCREAMING_SNAKE_CASE__								=      BertModel(snake_case__    )

			# Load weights from checkpoint
			logger.info(f"""Loading weights from checkpoint {tf_checkpoint_path}..."""    )
			load_tfa_weights_in_bert(snake_case__		,				snake_case__		,				snake_case__    )

			# Save pytorch-model
			logger.info(f"""Saving PyTorch model to {pytorch_dump_path}..."""    )
			torch.save(model.state_dict()		,				snake_case__    )


if __name__ == "__main__":
						A_							:			Optional[Any]											= argparse.ArgumentParser()
						parser.add_argument(
						    "--tf_checkpoint_path", type=str, required=True, help="Path to the TensorFlow 2.x checkpoint path."
						)
						parser.add_argument(
						    "--bert_config_file",
						    type=str,
						    required=True,
						    help="The config json file corresponding to the BERT model. This specifies the model architecture.",
						)
						parser.add_argument(
						    "--pytorch_dump_path",
						    type=str,
						    required=True,
						    help="Path to the output PyTorch model (must include filename).",
						)
						A_							:			Dict											= parser.parse_args()
						convert_tfa_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)





"""simple docstring"""




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_							:			List[Any]											= logging.get_logger(__name__)

A_							:			Dict											= {
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}





class    lowerCamelCase						(A__			):
			lowerCamelCase__   :		int							=							'unispeech'

			def __init__(					self				:						Union[str, Any]    ,				__UpperCAmelCase				:						List[Any]=3_2    ,				__UpperCAmelCase				:						Union[str, Any]=7_6_8    ,				__UpperCAmelCase				:						Tuple=1_2    ,				__UpperCAmelCase				:						Dict=1_2    ,				__UpperCAmelCase				:						Optional[Any]=3_0_7_2    ,				__UpperCAmelCase				:						Optional[Any]="gelu"    ,				__UpperCAmelCase				:						Union[str, Any]=0.1    ,				__UpperCAmelCase				:						List[Any]=0.1    ,				__UpperCAmelCase				:						List[str]=0.1    ,				__UpperCAmelCase				:						List[str]=0.0    ,				__UpperCAmelCase				:						Tuple=0.0    ,				__UpperCAmelCase				:						List[Any]=0.1    ,				__UpperCAmelCase				:						Union[str, Any]=0.1    ,				__UpperCAmelCase				:						Union[str, Any]=0.02    ,				__UpperCAmelCase				:						Union[str, Any]=1e-5    ,				__UpperCAmelCase				:						List[Any]="group"    ,				__UpperCAmelCase				:						Union[str, Any]="gelu"    ,				__UpperCAmelCase				:						Any=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)    ,				__UpperCAmelCase				:						List[str]=(5, 2, 2, 2, 2, 2, 2)    ,				__UpperCAmelCase				:						int=(1_0, 3, 3, 3, 3, 2, 2)    ,				__UpperCAmelCase				:						str=False    ,				__UpperCAmelCase				:						Any=1_2_8    ,				__UpperCAmelCase				:						str=1_6    ,				__UpperCAmelCase				:						Optional[Any]=False    ,				__UpperCAmelCase				:						Optional[int]=True    ,				__UpperCAmelCase				:						Union[str, Any]=0.05    ,				__UpperCAmelCase				:						str=1_0    ,				__UpperCAmelCase				:						Optional[Any]=2    ,				__UpperCAmelCase				:						List[Any]=0.0    ,				__UpperCAmelCase				:						Tuple=1_0    ,				__UpperCAmelCase				:						Tuple=0    ,				__UpperCAmelCase				:						Tuple=3_2_0    ,				__UpperCAmelCase				:						Any=2    ,				__UpperCAmelCase				:						Optional[int]=0.1    ,				__UpperCAmelCase				:						Any=1_0_0    ,				__UpperCAmelCase				:						str=2_5_6    ,				__UpperCAmelCase				:						Dict=2_5_6    ,				__UpperCAmelCase				:						Dict=0.1    ,				__UpperCAmelCase				:						List[str]="mean"    ,				__UpperCAmelCase				:						Optional[Any]=False    ,				__UpperCAmelCase				:						List[Any]=False    ,				__UpperCAmelCase				:						str=2_5_6    ,				__UpperCAmelCase				:						Dict=8_0    ,				__UpperCAmelCase				:						List[Any]=0    ,				__UpperCAmelCase				:						int=1    ,				__UpperCAmelCase				:						Optional[int]=2    ,				__UpperCAmelCase				:						Any=0.5    ,				**__UpperCAmelCase				:						List[str]    ,				)   ->    Tuple:
						super().__init__(**__UpperCAmelCase    ,				pad_token_id=__UpperCAmelCase    ,				bos_token_id=__UpperCAmelCase    ,				eos_token_id=__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      hidden_size
						SCREAMING_SNAKE_CASE__								=      feat_extract_norm
						SCREAMING_SNAKE_CASE__								=      feat_extract_activation
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      conv_bias
						SCREAMING_SNAKE_CASE__								=      num_conv_pos_embeddings
						SCREAMING_SNAKE_CASE__								=      num_conv_pos_embedding_groups
						SCREAMING_SNAKE_CASE__								=      len(self.conv_dim       )
						SCREAMING_SNAKE_CASE__								=      num_hidden_layers
						SCREAMING_SNAKE_CASE__								=      intermediate_size
						SCREAMING_SNAKE_CASE__								=      hidden_act
						SCREAMING_SNAKE_CASE__								=      num_attention_heads
						SCREAMING_SNAKE_CASE__								=      hidden_dropout
						SCREAMING_SNAKE_CASE__								=      attention_dropout
						SCREAMING_SNAKE_CASE__								=      activation_dropout
						SCREAMING_SNAKE_CASE__								=      feat_proj_dropout
						SCREAMING_SNAKE_CASE__								=      final_dropout
						SCREAMING_SNAKE_CASE__								=      layerdrop
						SCREAMING_SNAKE_CASE__								=      layer_norm_eps
						SCREAMING_SNAKE_CASE__								=      initializer_range
						SCREAMING_SNAKE_CASE__								=      num_ctc_classes
						SCREAMING_SNAKE_CASE__								=      vocab_size
						SCREAMING_SNAKE_CASE__								=      do_stable_layer_norm
						SCREAMING_SNAKE_CASE__								=      use_weighted_layer_sum
						SCREAMING_SNAKE_CASE__								=      classifier_proj_size

						if (
						    (len(self.conv_stride       ) != self.num_feat_extract_layers)
						    or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
						    or (len(self.conv_dim       ) != self.num_feat_extract_layers)
						):
									raise ValueError(
									    """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
									    """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
									    F""" {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,"""
									    F""" `len(config.conv_kernel) = {len(self.conv_kernel       )}`."""       )

						# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
						SCREAMING_SNAKE_CASE__								=      apply_spec_augment
						SCREAMING_SNAKE_CASE__								=      mask_time_prob
						SCREAMING_SNAKE_CASE__								=      mask_time_length
						SCREAMING_SNAKE_CASE__								=      mask_time_min_masks
						SCREAMING_SNAKE_CASE__								=      mask_feature_prob
						SCREAMING_SNAKE_CASE__								=      mask_feature_length
						SCREAMING_SNAKE_CASE__								=      mask_feature_min_masks

						# parameters for pretraining with codevector quantized representations
						SCREAMING_SNAKE_CASE__								=      num_codevectors_per_group
						SCREAMING_SNAKE_CASE__								=      num_codevector_groups
						SCREAMING_SNAKE_CASE__								=      contrastive_logits_temperature
						SCREAMING_SNAKE_CASE__								=      feat_quantizer_dropout
						SCREAMING_SNAKE_CASE__								=      num_negatives
						SCREAMING_SNAKE_CASE__								=      codevector_dim
						SCREAMING_SNAKE_CASE__								=      proj_codevector_dim
						SCREAMING_SNAKE_CASE__								=      diversity_loss_weight

						# ctc loss
						SCREAMING_SNAKE_CASE__								=      ctc_loss_reduction
						SCREAMING_SNAKE_CASE__								=      ctc_zero_infinity

						# pretraining loss
						SCREAMING_SNAKE_CASE__								=      replace_prob





			@property
			def 					SCREAMING_SNAKE_CASE				(					self				:						int       )   ->    Optional[Any]:
						return functools.reduce(operator.mul    ,				self.conv_stride    ,				1       )


