

# Usage:
# ./gen-card-allenai-wmt16.py

import os
from pathlib import Path






def 					lowerCamelCase_     (							UpperCAmelCase_							:     int							,       UpperCAmelCase_							:     str							,       UpperCAmelCase_							:     str							,       UpperCAmelCase_							:     Optional[Any]	)						->				Optional[int]:




	'''simple docstring'''




	_UpperCamelCase   :		Optional[int]			=						{
	    'en': 'Machine learning is great, isn\'t it?',
	    'ru': 'Машинное обучение - это здорово, не так ли?',
	    'de': 'Maschinelles Lernen ist großartig, nicht wahr?',
	}

	# BLUE scores as follows:
	# "pair": [fairseq, transformers]
	_UpperCamelCase   :		Optional[int]			=						{
	    'wmt16-en-de-dist-12-1': [2_8.3, 2_7.5_2],
	    'wmt16-en-de-dist-6-1': [2_7.4, 2_7.1_1],
	    'wmt16-en-de-12-1': [2_6.9, 2_5.7_5],
	}
	_UpperCamelCase   :		str			=						F'''{src_lang}-{tgt_lang}'''

	_UpperCamelCase   :		Tuple			=						F'''
---
language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt16
- allenai
license: apache-2.0
datasets:
- wmt16
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt16 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

All 3 models are available:

* [wmt16-en-de-dist-12-1](https://huggingface.co/allenai/wmt16-en-de-dist-12-1)
* [wmt16-en-de-dist-6-1](https://huggingface.co/allenai/wmt16-en-de-dist-6-1)
* [wmt16-en-de-12-1](https://huggingface.co/allenai/wmt16-en-de-12-1)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "allenai/{model_name}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   | fairseq | transformers
-------|---------|----------
{model_name}  | {scores[model_name][0]} | {scores[model_name][1]}

The score is slightly below the score reported in the paper, as the researchers don\'t use `sacrebleu` and measure the score on tokenized outputs. `transformers` score was measured using `sacrebleu` on detokenized outputs.

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt16 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt16 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt16/)
- [test set](http://matrix.statmt.org/test_sets/newstest2016.tgz?1504722372)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

'''
	model_card_dir.mkdir(parents=UpperCAmelCase_							,       exist_ok=UpperCAmelCase_	)
	_UpperCamelCase   :		Any			=						os.path.join(UpperCAmelCase_							,       'README.md'	)
	print(F'''Generating {path}'''	)
	with open(UpperCAmelCase_							,       'w'							,       encoding='utf-8'	) as f:
		f.write(UpperCAmelCase_	)


# make sure we are under the root of the project
lowerCAmelCase__								=		Path(__file__).resolve().parent.parent.parent
lowerCAmelCase__								=		repo_dir / """model_cards"""

for model_name in ["wmt16-en-de-dist-12-1", "wmt16-en-de-dist-6-1", "wmt16-en-de-12-1"]:
							lowerCAmelCase__								=		model_cards_dir / """allenai""" / model_name
							write_model_card(model_card_dir, src_lang="""en""", tgt_lang="""de""", model_name=model_name)







import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
							import tensorflow as tf

if is_tensorflow_text_available():
							from transformers.models.bert import TFBertTokenizer


lowerCAmelCase__								=		["""bert-base-uncased""", """bert-base-cased"""]
lowerCAmelCase__								=		"""hf-internal-testing/tiny-bert-tf-only"""

if is_tf_available():







							class 	lowercase    (				tf.keras.Model							):


									"""simple docstring"""




									def __init__(				self	,       __snake_case):
										super().__init__()
										_UpperCamelCase   :		List[Any]			=						tokenizer
										_UpperCamelCase   :		List[Any]			=						AutoConfig.from_pretrained(__snake_case)
										_UpperCamelCase   :		Dict			=						TFAutoModel.from_config(__snake_case)






									def 		A__					(				self	,       __snake_case):
										_UpperCamelCase   :		Any			=						self.tokenizer(__snake_case)
										_UpperCamelCase   :		Dict			=						self.bert(**__snake_case)
										return out["pooler_output"]







@require_tf
@require_tensorflow_text
class 	lowercase    (				unittest.TestCase							):


		"""simple docstring"""




		def 		A__					(				self):
			super().setUp()

			_UpperCamelCase   :		Optional[Any]			=						[
			    BertTokenizer.from_pretrained(__snake_case) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
			]  # repeat for when fast_bert_tokenizer=false
			_UpperCamelCase   :		Optional[Any]			=						[TFBertTokenizer.from_pretrained(__snake_case) for checkpoint in TOKENIZER_CHECKPOINTS] + [
			    TFBertTokenizer.from_pretrained(__snake_case	,       use_fast_bert_tokenizer=__snake_case)
			    for checkpoint in TOKENIZER_CHECKPOINTS
			]
			assert len(self.tokenizers) == len(self.tf_tokenizers)

			_UpperCamelCase   :		Optional[Any]			=						[
			    'This is a straightforward English test sentence.',
			    'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
			    'Now we\'re going to add some Chinese: 一 二 三 一二三',
			    'And some much more rare Chinese: 齉 堃 齉堃',
			    'Je vais aussi écrire en français pour tester les accents',
			    'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
			]
			_UpperCamelCase   :		Dict			=						list(zip(self.test_sentences	,       self.test_sentences[::-1]))


		def 		A__					(				self):
			for tokenizer, tf_tokenizer in zip(self.tokenizers	,       self.tf_tokenizers):
				for test_inputs in (self.test_sentences, self.paired_sentences):
					_UpperCamelCase   :		List[str]			=						tokenizer(__snake_case	,       return_tensors='tf'	,       padding='longest')
					_UpperCamelCase   :		Tuple			=						tf_tokenizer(__snake_case)

					for key in python_outputs.keys():
						self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape))
						self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]	,       tf.intaa) == tf_outputs[key]))


		@slow
		def 		A__					(				self):
			for tf_tokenizer in self.tf_tokenizers:
				_UpperCamelCase   :		Tuple			=						tf_tokenizer(self.paired_sentences)
				_UpperCamelCase   :		Optional[Any]			=						tf_tokenizer(
				    text=[sentence[0] for sentence in self.paired_sentences]	,       text_pair=[sentence[1] for sentence in self.paired_sentences]	,       )
				for key in merged_outputs.keys():
					self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]	,       tf.intaa) == separated_outputs[key]))


		@slow
		def 		A__					(				self):
			for tf_tokenizer in self.tf_tokenizers:
				_UpperCamelCase   :		Tuple			=						tf.function(__snake_case)
				for test_inputs in (self.test_sentences, self.paired_sentences):
					_UpperCamelCase   :		Optional[int]			=						tf.constant(__snake_case)
					_UpperCamelCase   :		Union[str, Any]			=						compiled_tokenizer(__snake_case)
					_UpperCamelCase   :		Tuple			=						tf_tokenizer(__snake_case)

					for key in eager_outputs.keys():
						self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]))






		@slow
		def 		A__					(				self):
			for tf_tokenizer in self.tf_tokenizers:
				_UpperCamelCase   :		Any			=						ModelToSave(tokenizer=__snake_case)
				_UpperCamelCase   :		Any			=						tf.convert_to_tensor(self.test_sentences)
				_UpperCamelCase   :		Union[str, Any]			=						model(__snake_case)  # Build model with some sample inputs
				with TemporaryDirectory() as tempdir:
					_UpperCamelCase   :		int			=						Path(__snake_case) / 'saved.model'
					model.save(__snake_case)
					_UpperCamelCase   :		Optional[int]			=						tf.keras.models.load_model(__snake_case)
				_UpperCamelCase   :		int			=						loaded_model(__snake_case)
				# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
				self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output))	,       1e-5)




