


"""simple docstring"""


import json
import os
import subprocess
import unittest
from ast import literal_eval

import pytest
from parameterized import parameterized, parameterized_class

from . import is_sagemaker_available


if is_sagemaker_available():
   from sagemaker import Session, TrainingJobAnalytics
   from sagemaker.huggingface import HuggingFace



@pytest.mark.skipif(
    literal_eval(os.getenv('TEST_SAGEMAKER'		,		'False' ) ) is not True		,		reason='Skipping test because should only be run when releasing minor transformers version'		,		)
@pytest.mark.usefixtures('sm_env' )
@parameterized_class(
    [
        {
            'framework': 'pytorch',
            'script': 'run_glue.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 650, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'pytorch',
            'script': 'run_ddp.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 600, 'eval_accuracy': 0.7, 'eval_loss': 0.6},
        },
        {
            'framework': 'tensorflow',
            'script': 'run_tf_dist.py',
            'model_name_or_path': 'distilbert-base-cased',
            'instance_type': 'ml.p3.16xlarge',
            'results': {'train_runtime': 600, 'eval_accuracy': 0.6, 'eval_loss': 0.7},
        },
    ] )
class   _a      (      unittest.TestCase ):
    def      _UpperCamelCase       ( self							:					Union[str, Any]				):







          if self.framework == "pytorch":
                subprocess.run(
                    F'cp ./examples/pytorch/text-classification/run_glue.py {self.env.test_path}/run_glue.py'.split()							,					encoding='utf-8'							,					check=lowerCAmelCase_							,					)
          assert hasattr(self							,					'env'				)
    def      _UpperCamelCase       ( self							:					Optional[Any]							,					SCREAMING_SNAKE_CASE__							:					Dict				):
          lowerCamelCase__           =		F'{self.env.base_job_name}-{instance_count}-{"ddp" if "ddp" in self.script else "smd"}'
          # distributed data settings
          lowerCamelCase__           =		{'smdistributed': {'dataparallel': {'enabled': True}}} if self.script != 'run_ddp.py' else None

          # creates estimator
          return HuggingFace(
              entry_point=self.script							,					source_dir=self.env.test_path							,					role=self.env.role							,					image_uri=self.env.image_uri							,					base_job_name=lowerCAmelCase_							,					instance_count=lowerCAmelCase_							,					instance_type=self.instance_type							,					debugger_hook_config=lowerCAmelCase_							,					hyperparameters={**self.env.distributed_hyperparameters, 'model_name_or_path': self.model_name_or_path}							,					metric_definitions=self.env.metric_definitions							,					distribution=lowerCAmelCase_							,					py_version='py36'							,					)
    def      _UpperCamelCase       ( self							:					int							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          TrainingJobAnalytics(lowerCAmelCase_				).export_csv(F'{self.env.test_path}/{job_name}_metrics.csv'				)


    @parameterized.expand([(2,)]				)
    def      _UpperCamelCase       ( self							:					str							,					SCREAMING_SNAKE_CASE__							:					List[Any]				):
          lowerCamelCase__           =		self.create_estimator(lowerCAmelCase_				)

          # run training
          estimator.fit()

          # result dataframe
          lowerCamelCase__           =		TrainingJobAnalytics(estimator.latest_training_job.name				).dataframe()

          # extract kpis
          lowerCamelCase__           =		list(result_metrics_df[result_metrics_df.metric_name == 'eval_accuracy']['value']				)
          lowerCamelCase__           =		list(result_metrics_df[result_metrics_df.metric_name == 'eval_loss']['value']				)
          # get train time from SageMaker job, this includes starting, preprocessing, stopping
          lowerCamelCase__           =		(
              Session().describe_training_job(estimator.latest_training_job.name				).get('TrainingTimeInSeconds'							,					99_99_99				)
          )

          # assert kpis
          assert train_runtime <= self.results["train_runtime"]
          assert all(t >= self.results['eval_accuracy'] for t in eval_accuracy				)
          assert all(t <= self.results['eval_loss'] for t in eval_loss				)

          # dump tests result into json file to share in PR
          with open(F'{estimator.latest_training_job.name}.json'							,					'w'				) as outfile:
                json.dump({'train_time': train_runtime, 'eval_accuracy': eval_accuracy, 'eval_loss': eval_loss}							,					lowerCAmelCase_				)







"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_snake_case          =							logging.get_logger(__name__)

_snake_case          =							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_snake_case          =							[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_snake_case          =							{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_snake_case          =							{f"""funnel-transformer/{name}""": 512 for name in _model_names}
_snake_case          =							{f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}



class   _a      (      SCREAMING_SNAKE_CASE_ ):
    a_    :				List[str]             =    VOCAB_FILES_NAMES
    a_    :				List[str]             =    PRETRAINED_VOCAB_FILES_MAP
    a_    :				Optional[int]             =    PRETRAINED_INIT_CONFIGURATION
    a_    :				List[str]             =    FunnelTokenizer
    a_    :				Dict             =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a_    :				int             =    2
    def __init__( self							:					int							,					SCREAMING_SNAKE_CASE__							:					Optional[int]=None							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]=None							,					SCREAMING_SNAKE_CASE__							:					int=True							,					SCREAMING_SNAKE_CASE__							:					Any="<unk>"							,					SCREAMING_SNAKE_CASE__							:					List[Any]="<sep>"							,					SCREAMING_SNAKE_CASE__							:					int="<pad>"							,					SCREAMING_SNAKE_CASE__							:					Tuple="<cls>"							,					SCREAMING_SNAKE_CASE__							:					Tuple="<mask>"							,					SCREAMING_SNAKE_CASE__							:					Any="<s>"							,					SCREAMING_SNAKE_CASE__							:					Tuple="</s>"							,					SCREAMING_SNAKE_CASE__							:					str=True							,					SCREAMING_SNAKE_CASE__							:					Any=True							,					SCREAMING_SNAKE_CASE__							:					str=None							,					SCREAMING_SNAKE_CASE__							:					int="##"							,					**SCREAMING_SNAKE_CASE__							:					Any							,					):
          super().__init__(
              SCREAMING_SNAKE_CASE__							,					tokenizer_file=SCREAMING_SNAKE_CASE__							,					do_lower_case=SCREAMING_SNAKE_CASE__							,					unk_token=SCREAMING_SNAKE_CASE__							,					sep_token=SCREAMING_SNAKE_CASE__							,					pad_token=SCREAMING_SNAKE_CASE__							,					cls_token=SCREAMING_SNAKE_CASE__							,					mask_token=SCREAMING_SNAKE_CASE__							,					bos_token=SCREAMING_SNAKE_CASE__							,					eos_token=SCREAMING_SNAKE_CASE__							,					clean_text=SCREAMING_SNAKE_CASE__							,					tokenize_chinese_chars=SCREAMING_SNAKE_CASE__							,					strip_accents=SCREAMING_SNAKE_CASE__							,					wordpieces_prefix=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__							,					)

          lowerCamelCase__           =		json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
          if (
              normalizer_state.get('lowercase'							,					SCREAMING_SNAKE_CASE__				) != do_lower_case
              or normalizer_state.get('strip_accents'							,					SCREAMING_SNAKE_CASE__				) != strip_accents
              or normalizer_state.get('handle_chinese_chars'							,					SCREAMING_SNAKE_CASE__				) != tokenize_chinese_chars
          ):
                lowerCamelCase__           =		getattr(SCREAMING_SNAKE_CASE__							,					normalizer_state.pop('type'				)				)
                lowerCamelCase__           =		do_lower_case
                lowerCamelCase__           =		strip_accents
                lowerCamelCase__           =		tokenize_chinese_chars
                lowerCamelCase__           =		normalizer_class(**SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		do_lower_case
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]=None				):
          lowerCamelCase__           =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					List[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[List[int]] = None				):
          lowerCamelCase__           =		[self.sep_token_id]
          lowerCamelCase__           =		[self.cls_token_id]
          if token_ids_a is None:
                return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
          return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]


    def      _UpperCamelCase       ( self							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					Optional[str] = None				):
          lowerCamelCase__           =		self._tokenizer.model.save(SCREAMING_SNAKE_CASE__							,					name=SCREAMING_SNAKE_CASE__				)
          return tuple(SCREAMING_SNAKE_CASE__				)



