"""simple docstring"""





import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class   UpperCAmelCase_			(					unittest.TestCase):
      lowerCamelCase__	:					Optional[int]										=						MODEL_FOR_MASKED_LM_MAPPING
      lowerCamelCase__	:					List[str]										=						TF_MODEL_FOR_MASKED_LM_MAPPING


      def 	_UpperCAmelCase (     self							) ->      List[str]:
             super().tearDown()
             # clean-up as much as possible GPU memory occupied by PyTorch
             gc.collect()
             if is_torch_available():
                    import torch

                    torch.cuda.empty_cache()


      @require_tf
      def 	_UpperCAmelCase (     self							) ->      List[str]:
             lowercase__    :	Dict     					=			pipeline(task='fill-mask'	,    model='sshleifer/tiny-distilroberta-base'	,    top_k=2	,    framework='tf'							)
             lowercase__    :	Tuple     					=			unmasker('My name is <mask>'							)
             self.assertEqual(
                 nested_simplify(a	,    decimals=6							)	,    [
                     {'sequence': 'My name is grouped', 'score': 2.1e-05, 'token': 3_8_0_1_5, 'token_str': ' grouped'},
                     {'sequence': 'My name is accuser', 'score': 2.1e-05, 'token': 2_5_5_0_6, 'token_str': ' accuser'},
                 ]	,    )

             lowercase__    :	Tuple     					=			unmasker('The largest city in France is <mask>'							)
             self.assertEqual(
                 nested_simplify(a	,    decimals=6							)	,    [
                     {
                         'sequence': 'The largest city in France is grouped',
                         'score': 2.1e-05,
                         'token': 3_8_0_1_5,
                         'token_str': ' grouped',
                     },
                     {
                         'sequence': 'The largest city in France is accuser',
                         'score': 2.1e-05,
                         'token': 2_5_5_0_6,
                         'token_str': ' accuser',
                     },
                 ]	,    )

             lowercase__    :	str     					=			unmasker('My name is <mask>'	,    targets=[' Patrick', ' Clara', ' Teven']	,    top_k=3							)
             self.assertEqual(
                 nested_simplify(a	,    decimals=6							)	,    [
                     {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 1_3_6_0_6, 'token_str': ' Clara'},
                     {'sequence': 'My name is Patrick', 'score': 2e-05, 'token': 3_4_9_9, 'token_str': ' Patrick'},
                     {'sequence': 'My name is Te', 'score': 1.9e-05, 'token': 2_9_4_1, 'token_str': ' Te'},
                 ]	,    )


      @require_torch
      def 	_UpperCAmelCase (     self							) ->      List[str]:
             lowercase__    :	int     					=			pipeline(task='fill-mask'	,    model='sshleifer/tiny-distilroberta-base'	,    top_k=2	,    framework='pt'							)

             lowercase__    :	Optional[Any]     					=			unmasker('My name is <mask>'							)
             self.assertEqual(
                 nested_simplify(a	,    decimals=6							)	,    [
                     {'sequence': 'My name is Maul', 'score': 2.2e-05, 'token': 3_5_6_7_6, 'token_str': ' Maul'},
                     {'sequence': 'My name isELS', 'score': 2.2e-05, 'token': 1_6_4_1_6, 'token_str': 'ELS'},
                 ]	,    )

             lowercase__    :	Tuple     					=			unmasker('The largest city in France is <mask>'							)
             self.assertEqual(
                 nested_simplify(a	,    decimals=6							)	,    [
                     {
                         'sequence': 'The largest city in France is Maul',
                         'score': 2.2e-05,
                         'token': 3_5_6_7_6,
                         'token_str': ' Maul',
                     },
                     {'sequence': 'The largest city in France isELS', 'score': 2.2e-05, 'token': 1_6_4_1_6, 'token_str': 'ELS'},
                 ]	,    )

             lowercase__    :	Any     					=			unmasker('My name is <mask>'	,    targets=[' Patrick', ' Clara', ' Teven']	,    top_k=3							)
             self.assertEqual(
                 nested_simplify(a	,    decimals=6							)	,    [
                     {'sequence': 'My name is Patrick', 'score': 2.1e-05, 'token': 3_4_9_9, 'token_str': ' Patrick'},
                     {'sequence': 'My name is Te', 'score': 2e-05, 'token': 2_9_4_1, 'token_str': ' Te'},
                     {'sequence': 'My name is Clara', 'score': 2e-05, 'token': 1_3_6_0_6, 'token_str': ' Clara'},
                 ]	,    )

             lowercase__    :	List[Any]     					=			unmasker('My name is <mask> <mask>'	,    top_k=2							)

             self.assertEqual(
                 nested_simplify(a	,    decimals=6							)	,    [
                     [
                         {
                             'score': 2.2e-05,
                             'token': 3_5_6_7_6,
                             'token_str': ' Maul',
                             'sequence': '<s>My name is Maul<mask></s>',
                         },
                         {'score': 2.2e-05, 'token': 1_6_4_1_6, 'token_str': 'ELS', 'sequence': '<s>My name isELS<mask></s>'},
                     ],
                     [
                         {
                             'score': 2.2e-05,
                             'token': 3_5_6_7_6,
                             'token_str': ' Maul',
                             'sequence': '<s>My name is<mask> Maul</s>',
                         },
                         {'score': 2.2e-05, 'token': 1_6_4_1_6, 'token_str': 'ELS', 'sequence': '<s>My name is<mask>ELS</s>'},
                     ],
                 ]	,    )


      @require_torch_gpu
      def 	_UpperCAmelCase (     self							) ->      str:
             lowercase__    :	str     					=			pipeline('fill-mask'	,    model='hf-internal-testing/tiny-random-distilbert'	,    device=0	,    framework='pt'							)

             # convert model to fp16
             pipe.model.half()

             lowercase__    :	List[Any]     					=			pipe('Paris is the [MASK] of France.'							)
             # We actually don't care about the result, we just want to make sure
             # it works, meaning the float16 tensor got casted back to float32
             # for postprocessing.
             self.assertIsInstance(a	,    a							)


      @slow
      @require_torch
      def 	_UpperCAmelCase (     self							) ->      str:
             lowercase__    :	Optional[Any]     					=			pipeline(task='fill-mask'	,    model='distilroberta-base'	,    top_k=2	,    framework='pt'							)
             self.run_large_test(a							)


      @slow
      @require_tf
      def 	_UpperCAmelCase (     self							) ->      Optional[Any]:
             lowercase__    :	Tuple     					=			pipeline(task='fill-mask'	,    model='distilroberta-base'	,    top_k=2	,    framework='tf'							)
             self.run_large_test(a							)


      def 	_UpperCAmelCase (     self	,    a							) ->      Dict:
             lowercase__    :	Dict     					=			unmasker('My name is <mask>'							)
             self.assertEqual(
                 nested_simplify(a							)	,    [
                     {'sequence': 'My name is John', 'score': 0.008, 'token': 6_1_0, 'token_str': ' John'},
                     {'sequence': 'My name is Chris', 'score': 0.007, 'token': 1_5_7_3, 'token_str': ' Chris'},
                 ]	,    )
             lowercase__    :	Any     					=			unmasker('The largest city in France is <mask>'							)
             self.assertEqual(
                 nested_simplify(a							)	,    [
                     {
                         'sequence': 'The largest city in France is Paris',
                         'score': 0.251,
                         'token': 2_2_0_1,
                         'token_str': ' Paris',
                     },
                     {
                         'sequence': 'The largest city in France is Lyon',
                         'score': 0.214,
                         'token': 1_2_7_9_0,
                         'token_str': ' Lyon',
                     },
                 ]	,    )

             lowercase__    :	Tuple     					=			unmasker('My name is <mask>'	,    targets=[' Patrick', ' Clara', ' Teven']	,    top_k=3							)
             self.assertEqual(
                 nested_simplify(a							)	,    [
                     {'sequence': 'My name is Patrick', 'score': 0.005, 'token': 3_4_9_9, 'token_str': ' Patrick'},
                     {'sequence': 'My name is Clara', 'score': 0.000, 'token': 1_3_6_0_6, 'token_str': ' Clara'},
                     {'sequence': 'My name is Te', 'score': 0.000, 'token': 2_9_4_1, 'token_str': ' Te'},
                 ]	,    )


      @require_torch
      def 	_UpperCAmelCase (     self							) ->      str:
             lowercase__    :	int     					=			pipeline(task='fill-mask'	,    model='sshleifer/tiny-distilroberta-base'	,    framework='pt'							)
             lowercase__    :	int     					=			None
             lowercase__    :	List[str]     					=			None
             self.run_pipeline_test(a	,    []							)


      @require_tf
      def 	_UpperCAmelCase (     self							) ->      Optional[int]:
             lowercase__    :	Optional[int]     					=			pipeline(task='fill-mask'	,    model='sshleifer/tiny-distilroberta-base'	,    framework='tf'							)
             lowercase__    :	Union[str, Any]     					=			None
             lowercase__    :	Any     					=			None
             self.run_pipeline_test(a	,    []							)


      def 	_UpperCAmelCase (     self	,    a	,    a	,    a							) ->      Optional[int]:
             if tokenizer is None or tokenizer.mask_token_id is None:
                    self.skipTest('The provided tokenizer has no mask token, (probably reformer or wav2vec2)'							)

             lowercase__    :	Optional[Any]     					=			FillMaskPipeline(model=a	,    tokenizer=a							)
             lowercase__    :	Dict     					=			[
                 f"""This is another {tokenizer.mask_token} test""",
             ]
             return fill_masker, examples


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      Any:
             lowercase__    :	List[Any]     					=			fill_masker.tokenizer
             lowercase__    :	Optional[int]     					=			fill_masker.model

             lowercase__    :	Tuple     					=			fill_masker(
                 f"""This is a {tokenizer.mask_token}"""	,    )
             self.assertEqual(
                 a	,    [
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                 ]	,    )

             lowercase__    :	List[Any]     					=			fill_masker([f"""This is a {tokenizer.mask_token}"""]							)
             self.assertEqual(
                 a	,    [
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                 ]	,    )

             lowercase__    :	Optional[int]     					=			fill_masker([f"""This is a {tokenizer.mask_token}""", f"""Another {tokenizer.mask_token} great test."""]							)
             self.assertEqual(
                 a	,    [
                     [
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     ],
                     [
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     ],
                 ]	,    )

             with self.assertRaises(a							):
                    fill_masker([None]							)
             # No mask_token is not supported
             with self.assertRaises(a							):
                    fill_masker('This is'							)

             self.run_test_top_k(a	,    a							)
             self.run_test_targets(a	,    a							)
             self.run_test_top_k_targets(a	,    a							)
             self.fill_mask_with_duplicate_targets_and_top_k(a	,    a							)
             self.fill_mask_with_multiple_masks(a	,    a							)


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      Tuple:
             lowercase__    :	Optional[int]     					=			tokenizer.get_vocab()
             lowercase__    :	Dict     					=			sorted(vocab.keys()							)[:2]
             # Pipeline argument
             lowercase__    :	Dict     					=			FillMaskPipeline(model=a	,    tokenizer=a	,    targets=a							)
             lowercase__    :	Union[str, Any]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""							)
             self.assertEqual(
                 a	,    [
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                 ]	,    )
             lowercase__    :	Any     					=			{vocab[el] for el in targets}
             self.assertEqual({el['token'] for el in outputs}	,    a							)
             lowercase__    :	Optional[Any]     					=			[tokenizer.decode([x]							) for x in target_ids]
             self.assertEqual({el['token_str'] for el in outputs}	,    set(a							)							)

             # Call argument
             lowercase__    :	str     					=			FillMaskPipeline(model=a	,    tokenizer=a							)
             lowercase__    :	str     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    targets=a							)
             self.assertEqual(
                 a	,    [
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                 ]	,    )
             lowercase__    :	Dict     					=			{vocab[el] for el in targets}
             self.assertEqual({el['token'] for el in outputs}	,    a							)
             lowercase__    :	Any     					=			[tokenizer.decode([x]							) for x in target_ids]
             self.assertEqual({el['token_str'] for el in outputs}	,    set(a							)							)

             # Score equivalence
             lowercase__    :	List[Any]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    targets=a							)
             lowercase__    :	Optional[int]     					=			[top_mask['token_str'] for top_mask in outputs]
             lowercase__    :	int     					=			[top_mask['score'] for top_mask in outputs]

             # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
             if set(a							) == set(a							):
                    lowercase__    :	Optional[int]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    targets=a							)
                    lowercase__    :	Dict     					=			[top_mask['score'] for top_mask in unmasked_targets]
                    self.assertEqual(nested_simplify(a							)	,    nested_simplify(a							)							)

             # Raises with invalid
             with self.assertRaises(a							):
                    lowercase__    :	Optional[int]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    targets=[]							)
             # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
             if "" not in tokenizer.get_vocab():
                    with self.assertRaises(a							):
                           lowercase__    :	Dict     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    targets=['']							)
                    with self.assertRaises(a							):
                           lowercase__    :	Optional[Any]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    targets=''							)


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      int:
             lowercase__    :	List[str]     					=			FillMaskPipeline(model=a	,    tokenizer=a	,    top_k=2							)
             lowercase__    :	Optional[int]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""							)
             self.assertEqual(
                 a	,    [
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                 ]	,    )

             lowercase__    :	List[Any]     					=			FillMaskPipeline(model=a	,    tokenizer=a							)
             lowercase__    :	List[str]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    top_k=2							)
             self.assertEqual(
                 a	,    [
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                 ]	,    )
             self.assertEqual(nested_simplify(a							)	,    nested_simplify(a							)							)


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      List[str]:
             lowercase__    :	Union[str, Any]     					=			tokenizer.get_vocab()
             lowercase__    :	Dict     					=			FillMaskPipeline(model=a	,    tokenizer=a							)

             # top_k=2, ntargets=3
             lowercase__    :	Union[str, Any]     					=			sorted(vocab.keys()							)[:3]
             lowercase__    :	int     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    top_k=2	,    targets=a							)

             # If we use the most probably targets, and filter differently, we should still
             # have the same results
             lowercase__    :	Optional[int]     					=			[el['token_str'] for el in sorted(a	,    key=lambda a							: x["score"]	,    reverse=a							)]
             # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
             if set(a							).issubset(a							):
                    lowercase__    :	Optional[int]     					=			fill_masker(f"""This is a {tokenizer.mask_token}"""	,    top_k=3	,    targets=a							)
                    # They should yield exactly the same result
                    self.assertEqual(nested_simplify(a							)	,    nested_simplify(a							)							)


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      List[str]:
             lowercase__    :	Dict     					=			FillMaskPipeline(model=a	,    tokenizer=a							)
             lowercase__    :	int     					=			tokenizer.get_vocab()
             # String duplicates + id duplicates
             lowercase__    :	Optional[Any]     					=			sorted(vocab.keys()							)[:3]
             lowercase__    :	Tuple     					=			[targets[0], targets[1], targets[0], targets[2], targets[1]]
             lowercase__    :	Dict     					=			fill_masker(f"""My name is {tokenizer.mask_token}"""	,    targets=a	,    top_k=1_0							)

             # The target list contains duplicates, so we can't output more
             # than them
             self.assertEqual(len(a							)	,    3							)


      def 	_UpperCAmelCase (     self	,    a	,    a							) ->      List[str]:
             lowercase__    :	List[Any]     					=			FillMaskPipeline(model=a	,    tokenizer=a							)

             lowercase__    :	Tuple     					=			fill_masker(
                 f"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""	,    top_k=2							)
             self.assertEqual(
                 a	,    [
                     [
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     ],
                     [
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     ],
                     [
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                         {'sequence': ANY(a							), 'score': ANY(a							), 'token': ANY(a							), 'token_str': ANY(a							)},
                     ],
                 ]	,    )


"""simple docstring"""





import numpy as np
from sklearn.datasets import fetch_california_housing
from sklearn.metrics import mean_absolute_error, mean_squared_error
from sklearn.model_selection import train_test_split
from xgboost import XGBRegressor





def        a_  (   _lowerCAmelCase						:				dict  ):





       '''simple docstring'''

       return (data["data"], data["target"])





def        a_  (   _lowerCAmelCase						:				np.ndarray     ,				_lowerCAmelCase						:				np.ndarray     ,				_lowerCAmelCase						:				np.ndarray  ):





       '''simple docstring'''

       lowercase__    :	Any     					=			XGBRegressor(verbosity=0     ,				random_state=42  )
       xgb.fit(_lowerCAmelCase     ,				_lowerCAmelCase  )
       # Predict target for test data
       lowercase__    :	str     					=			xgb.predict(_lowerCAmelCase  )
       lowercase__    :	Union[str, Any]     					=			predictions.reshape(len(_lowerCAmelCase  )     ,				1  )
       return predictions





def        a_  (   ):





       '''simple docstring'''

       lowercase__    :	Optional[Any]     					=			fetch_california_housing()
       lowercase__					,   lowercase__    :	str     					=			data_handling(_lowerCAmelCase  )
       lowercase__					,   lowercase__					,   lowercase__					,   lowercase__    :	str     					=			train_test_split(
           _lowerCAmelCase     ,				_lowerCAmelCase     ,				test_size=0.2_5     ,				random_state=1  )
       lowercase__    :	Any     					=			xgboost(_lowerCAmelCase     ,				_lowerCAmelCase     ,				_lowerCAmelCase  )
       # Error printing
       print(f"""Mean Absolute Error : {mean_absolute_error(_lowerCAmelCase     ,				_lowerCAmelCase  )}"""  )
       print(f"""Mean Square Error  : {mean_squared_error(_lowerCAmelCase     ,				_lowerCAmelCase  )}"""  )


if __name__ == "__main__":
       import doctest

       doctest.testmod(verbose=True)
       main()

