# This script creates a super tiny model that is useful inside tests, when we just want to test that
# the machinery works, without needing to the check the quality of the outcomes.
#
# This version creates a tiny model through reduction of a normal pre-trained model, but keeping the
# full vocab, merges file, and thus also resulting in a larger model due to a large vocab size.
# This gives ~3MB in total for all files.
#
# If you want a 50 times smaller than this see `fsmt-make-super-tiny-model.py`, which is slightly more complicated
#
#
# It will be used then as "stas/tiny-wmt19-en-de"

# Build
from transformers import FSMTTokenizer, FSMTConfig, FSMTForConditionalGeneration

_lowerCamelCase					:   Union[str, Any]							=		'''facebook/wmt19-en-de'''
_lowerCamelCase					:   Optional[int]							=		FSMTTokenizer.from_pretrained(mname)
# get the correct vocab sizes, etc. from the master model
_lowerCamelCase					:   Optional[int]							=		FSMTConfig.from_pretrained(mname)
config.update(
    dict(
        d_model=4,
        encoder_layers=1,
        decoder_layers=1,
        encoder_ffn_dim=4,
        decoder_ffn_dim=4,
        encoder_attention_heads=1,
        decoder_attention_heads=1,
    )
)

_lowerCamelCase					:   Optional[int]							=		FSMTForConditionalGeneration(config)
print(f"""num of params {tiny_model.num_parameters()}""")

# Test
_lowerCamelCase					:   Optional[Any]							=		tokenizer(['''Making tiny model'''], return_tensors='''pt''')
_lowerCamelCase					:   Dict							=		tiny_model(**batch)

print('''test output:''', len(outputs.logits[0]))

# Save
_lowerCamelCase					:   Union[str, Any]							=		'''tiny-wmt19-en-de'''
tiny_model.half()  # makes it smaller
tiny_model.save_pretrained(mname_tiny)
tokenizer.save_pretrained(mname_tiny)

print(f"""Generated {mname_tiny}""")

# Upload
# transformers-cli upload tiny-wmt19-en-de





import argparse
import logging
from collections import namedtuple

import torch
from model_bertabs import BertAbsSummarizer
from models.model_builder import AbsSummarizer  # The authors' implementation

from transformers import BertTokenizer


logging.basicConfig(level=logging.INFO)
_lowerCamelCase					:   Optional[int]							=		logging.getLogger(__name__)


_lowerCamelCase					:   Optional[int]							=		'''Hello world! cécé herlolip'''


_lowerCamelCase					:   List[Any]							=		namedtuple(
    '''BertAbsConfig''',
    [
        '''temp_dir''',
        '''large''',
        '''use_bert_emb''',
        '''finetune_bert''',
        '''encoder''',
        '''share_emb''',
        '''max_pos''',
        '''enc_layers''',
        '''enc_hidden_size''',
        '''enc_heads''',
        '''enc_ff_size''',
        '''enc_dropout''',
        '''dec_layers''',
        '''dec_hidden_size''',
        '''dec_heads''',
        '''dec_ff_size''',
        '''dec_dropout''',
    ],
)





def 	__lowerCamelCase							(UpperCAmelCase__     :							str      , UpperCAmelCase__     :							Optional[int]  ):
					SCREAMING_SNAKE_CASE    			=       BertAbsConfig(
					    temp_dir="."      , finetune_bert=UpperCAmelCase__      , large=UpperCAmelCase__      , share_emb=UpperCAmelCase__      , use_bert_emb=UpperCAmelCase__      , encoder="bert"      , max_pos=5_1_2      , enc_layers=6      , enc_hidden_size=5_1_2      , enc_heads=8      , enc_ff_size=5_1_2      , enc_dropout=0.2      , dec_layers=6      , dec_hidden_size=7_6_8      , dec_heads=8      , dec_ff_size=2_0_4_8      , dec_dropout=0.2      , )
					SCREAMING_SNAKE_CASE    			=       torch.load(UpperCAmelCase__      , lambda UpperCAmelCase__      , UpperCAmelCase__  : storage  )
					SCREAMING_SNAKE_CASE    			=       AbsSummarizer(UpperCAmelCase__      , torch.device("cpu"  )      , UpperCAmelCase__  )
					original.eval()

					SCREAMING_SNAKE_CASE    			=       BertAbsSummarizer(UpperCAmelCase__      , torch.device("cpu"  )  )
					new_model.eval()

					# -------------------
					# Convert the weights
					# -------------------

					logging.info("convert the model"  )
					new_model.bert.load_state_dict(original.bert.state_dict()  )
					new_model.decoder.load_state_dict(original.decoder.state_dict()  )
					new_model.generator.load_state_dict(original.generator.state_dict()  )

					# ----------------------------------
					# Make sure the outpus are identical
					# ----------------------------------

					logging.info("Make sure that the models' outputs are identical"  )
					SCREAMING_SNAKE_CASE    			=       BertTokenizer.from_pretrained("bert-base-uncased"  )

					# prepare the model inputs
					SCREAMING_SNAKE_CASE    			=       tokenizer.encode("This is sample éàalj'-."  )
					encoder_input_ids.extend([tokenizer.pad_token_id] * (5_1_2 - len(UpperCAmelCase__  ))  )
					SCREAMING_SNAKE_CASE    			=       torch.tensor(UpperCAmelCase__  ).unsqueeze(0  )
					SCREAMING_SNAKE_CASE    			=       tokenizer.encode("This is sample 3 éàalj'-."  )
					decoder_input_ids.extend([tokenizer.pad_token_id] * (5_1_2 - len(UpperCAmelCase__  ))  )
					SCREAMING_SNAKE_CASE    			=       torch.tensor(UpperCAmelCase__  ).unsqueeze(0  )

					# failsafe to make sure the weights reset does not affect the
					# loaded weights.
					assert torch.max(torch.abs(original.generator[0].weight - new_model.generator[0].weight  )  ) == 0

					# forward pass
					SCREAMING_SNAKE_CASE    			=       encoder_input_ids
					SCREAMING_SNAKE_CASE    			=       decoder_input_ids
					SCREAMING_SNAKE_CASE    			=       SCREAMING_SNAKE_CASE    			=       None
					SCREAMING_SNAKE_CASE    			=       None
					SCREAMING_SNAKE_CASE    			=       SCREAMING_SNAKE_CASE    			=       None
					SCREAMING_SNAKE_CASE    			=       SCREAMING_SNAKE_CASE    			=       None
					SCREAMING_SNAKE_CASE    			=       None

					# The original model does not apply the geneator layer immediatly but rather in
					# the beam search (where it combines softmax + linear layer). Since we already
					# apply the softmax in our generation process we only apply the linear layer here.
					# We make sure that the outputs of the full stack are identical
					SCREAMING_SNAKE_CASE    			=       original(UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__  )[0]
					SCREAMING_SNAKE_CASE    			=       original.generator(UpperCAmelCase__  )

					SCREAMING_SNAKE_CASE    			=       new_model(
					    UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__      , UpperCAmelCase__  )[0]
					SCREAMING_SNAKE_CASE    			=       new_model.generator(UpperCAmelCase__  )

					SCREAMING_SNAKE_CASE    			=       torch.max(torch.abs(output_converted_model - output_original_model  )  ).item()
					print("Maximum absolute difference beween weights: {:.2f}".format(UpperCAmelCase__  )  )
					SCREAMING_SNAKE_CASE    			=       torch.max(torch.abs(output_converted_generator - output_original_generator  )  ).item()
					print("Maximum absolute difference beween weights: {:.2f}".format(UpperCAmelCase__  )  )

					SCREAMING_SNAKE_CASE    			=       torch.allclose(UpperCAmelCase__      , UpperCAmelCase__      , atol=1e-3  )
					if are_identical:
										logging.info("all weights are equal up to 1e-3"  )
					else:
										raise ValueError("the weights are different. The new model is likely different from the original one."  )

					# The model has been saved with torch.save(model) and this is bound to the exact
					# directory structure. We save the state_dict instead.
					logging.info("saving the model's state dictionary"  )
					torch.save(
					    new_model.state_dict()      , "./bertabs-finetuned-cnndm-extractive-abstractive-summarization/pytorch_model.bin"  )


if __name__ == "__main__":
							_lowerCamelCase					:   Union[str, Any]							=		argparse.ArgumentParser()
							parser.add_argument(
							    '''--bertabs_checkpoint_path''',
							    default=None,
							    type=str,
							    required=True,
							    help='''Path the official PyTorch dump.''',
							)
							parser.add_argument(
							    '''--pytorch_dump_folder_path''',
							    default=None,
							    type=str,
							    required=True,
							    help='''Path to the output PyTorch model.''',
							)
							_lowerCamelCase					:   Any							=		parser.parse_args()

							convert_bertabs_checkpoints(
							    args.bertabs_checkpoint_path,
							    args.pytorch_dump_folder_path,
							)




