





import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import datasets
import numpy as np
import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    EvalPrediction,
    HfArgumentParser,
    PreTrainedTokenizer,
    TFAutoModelForSequenceClassification,
    TFTrainer,
    TFTrainingArguments,
)
from transformers.utils import logging as hf_logging


hf_logging.set_verbosity_info()
hf_logging.enable_default_handler()
hf_logging.enable_explicit_format()


def     lowerCamelCase_	(      lowerCAmelCase:	str    ,							lowerCAmelCase:	str    ,							lowerCAmelCase:	str    ,							lowerCAmelCase:	PreTrainedTokenizer    ,							lowerCAmelCase:	int    ,							lowerCAmelCase:	Optional[int] = None    ,							)->		Optional[Any]:
  _snake_case   :				str														=	{}

  if train_file is not None:
    _snake_case   :				Optional[Any]														=	[train_file]
  if eval_file is not None:
    _snake_case   :				int														=	[eval_file]
  if test_file is not None:
    _snake_case   :				int														=	[test_file]

  _snake_case   :				Optional[int]														=	datasets.load_dataset('csv'    ,							data_files=_lowerCamelCase				)
  _snake_case   :				List[str]														=	list(ds[list(files.keys()				)[0]].features.keys()				)
  _snake_case   :				int														=	features_name.pop(_lowerCamelCase				)
  _snake_case   :				Optional[Any]														=	list(set(ds[list(files.keys()				)[0]][label_name]				)				)
  _snake_case   :				List[str]														=	{label: i for i, label in enumerate(_lowerCamelCase				)}
  _snake_case   :				Optional[int]														=	tokenizer.model_input_names
  _snake_case   :				Any														=	{}

  if len(_lowerCamelCase				) == 1:
    for k in files.keys():
      _snake_case   :				List[Any]														=	ds[k].map(
          lambda lowerCAmelCase				: tokenizer.batch_encode_plus(
              example[features_name[0]]    ,							truncation=_lowerCamelCase    ,							max_length=_lowerCamelCase    ,							padding='max_length'				)    ,							batched=_lowerCamelCase    ,							)
  elif len(_lowerCamelCase				) == 2:
    for k in files.keys():
      _snake_case   :				Any														=	ds[k].map(
          lambda lowerCAmelCase				: tokenizer.batch_encode_plus(
              (example[features_name[0]], example[features_name[1]])    ,							truncation=_lowerCamelCase    ,							max_length=_lowerCamelCase    ,							padding='max_length'    ,							)    ,							batched=_lowerCamelCase    ,							)

  def gen_train():
    for ex in transformed_ds[datasets.Split.TRAIN]:
      _snake_case   :				Tuple														=	{k: v for k, v in ex.items() if k in input_names}
      _snake_case   :				List[Any]														=	labelaid[ex[label_name]]
      yield (d, label)

  def gen_val():
    for ex in transformed_ds[datasets.Split.VALIDATION]:
      _snake_case   :				int														=	{k: v for k, v in ex.items() if k in input_names}
      _snake_case   :				Optional[int]														=	labelaid[ex[label_name]]
      yield (d, label)

  def gen_test():
    for ex in transformed_ds[datasets.Split.TEST]:
      _snake_case   :				int														=	{k: v for k, v in ex.items() if k in input_names}
      _snake_case   :				Optional[Any]														=	labelaid[ex[label_name]]
      yield (d, label)

  _snake_case   :				Tuple														=	(
      tf.data.Dataset.from_generator(
          _lowerCamelCase    ,							({k: tf.intaa for k in input_names}, tf.intaa)    ,							({k: tf.TensorShape([None]				) for k in input_names}, tf.TensorShape([]				))    ,							)
      if datasets.Split.TRAIN in transformed_ds
      else None
  )

  if train_ds is not None:
    _snake_case   :				Any														=	train_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TRAIN]				)				)				)

  _snake_case   :				Optional[Any]														=	(
      tf.data.Dataset.from_generator(
          _lowerCamelCase    ,							({k: tf.intaa for k in input_names}, tf.intaa)    ,							({k: tf.TensorShape([None]				) for k in input_names}, tf.TensorShape([]				))    ,							)
      if datasets.Split.VALIDATION in transformed_ds
      else None
  )

  if val_ds is not None:
    _snake_case   :				Union[str, Any]														=	val_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.VALIDATION]				)				)				)

  _snake_case   :				Dict														=	(
      tf.data.Dataset.from_generator(
          _lowerCamelCase    ,							({k: tf.intaa for k in input_names}, tf.intaa)    ,							({k: tf.TensorShape([None]				) for k in input_names}, tf.TensorShape([]				))    ,							)
      if datasets.Split.TEST in transformed_ds
      else None
  )

  if test_ds is not None:
    _snake_case   :				Dict														=	test_ds.apply(tf.data.experimental.assert_cardinality(len(ds[datasets.Split.TEST]				)				)				)

  return train_ds, val_ds, test_ds, labelaid


lowerCAmelCase_						   =    logging.getLogger(__name__)



@dataclass
class        _lowerCAmelCase    :


  '''simple docstring'''
  a_							:       Optional[int]      					=field(metadata={"""help""": """Which column contains the label"""}							)
  a_							:       int      					=field(default=lowercase__							,    metadata={"""help""": """The path of the training file"""}							)
  a_							:       Dict      					=field(default=lowercase__							,    metadata={"""help""": """The path of the development file"""}							)
  a_							:       Tuple      					=field(default=lowercase__							,    metadata={"""help""": """The path of the test file"""}							)
  a_							:       Optional[int]      					=field(
      default=128							,    metadata={
          """help""": (
              """The maximum total input sequence length after tokenization. Sequences longer """
              """than this will be truncated, sequences shorter will be padded."""
          )
      }							,    )
  a_							:       Any      					=field(
      default=lowercase__							,    metadata={"""help""": """Overwrite the cached training and evaluation sets"""}							)




@dataclass
class        _lowerCAmelCase    :


  '''simple docstring'''
  a_							:       Optional[int]      					=field(
      metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}							)
  a_							:       Tuple      					=field(
      default=lowercase__							,    metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}							)
  a_							:       List[str]      					=field(
      default=lowercase__							,    metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}							)
  a_							:       Union[str, Any]      					=field(default=lowercase__							,    metadata={"""help""": """Set this flag to use fast tokenization."""}							)
  # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
  # or just modify its tokenizer_config.json.
  a_							:       Tuple      					=field(
      default=lowercase__							,    metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}							,    )




def     lowerCamelCase_	(      )->		int:
  _snake_case   :				Union[str, Any]														=	HfArgumentParser((ModelArguments, DataTrainingArguments, TFTrainingArguments)				)
  _snake_case   :				List[Any]														=	parser.parse_args_into_dataclasses()

  if (
      os.path.exists(training_args.output_dir				)
      and os.listdir(training_args.output_dir				)
      and training_args.do_train
      and not training_args.overwrite_output_dir
  ):
    raise ValueError(
        F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
        ' --overwrite_output_dir to overcome.'				)

  # Setup logging
  logging.basicConfig(
      format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'    ,							datefmt='%m/%d/%Y %H:%M:%S'    ,							level=logging.INFO    ,							)
  logger.info(
      F"""n_replicas: {training_args.n_replicas}, distributed training: {bool(training_args.n_replicas > 1				)}, """
      F"""16-bits training: {training_args.fpaa}"""				)
  logger.info(F"""Training/evaluation parameters {training_args}"""				)

  # Load pretrained model and tokenizer
  #
  # Distributed training:
  # The .from_pretrained methods guarantee that only one local process can concurrently
  # download model & vocab.

  _snake_case   :				Any														=	AutoTokenizer.from_pretrained(
      model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path    ,							cache_dir=model_args.cache_dir    ,							)

  _snake_case   :				Union[str, Any]														=	get_tfds(
      train_file=data_args.train_file    ,							eval_file=data_args.dev_file    ,							test_file=data_args.test_file    ,							tokenizer=_lowerCamelCase    ,							label_column_id=data_args.label_column_id    ,							max_seq_length=data_args.max_seq_length    ,							)

  _snake_case   :				str														=	AutoConfig.from_pretrained(
      model_args.config_name if model_args.config_name else model_args.model_name_or_path    ,							num_labels=len(_lowerCamelCase				)    ,							labelaid=_lowerCamelCase    ,							idalabel={id: label for label, id in labelaid.items()}    ,							finetuning_task='text-classification'    ,							cache_dir=model_args.cache_dir    ,							)

  with training_args.strategy.scope():
    _snake_case   :				Optional[Any]														=	TFAutoModelForSequenceClassification.from_pretrained(
        model_args.model_name_or_path    ,							from_pt=bool('.bin' in model_args.model_name_or_path				)    ,							config=_lowerCamelCase    ,							cache_dir=model_args.cache_dir    ,							)

  def compute_metrics(lowerCAmelCase:	EvalPrediction				) -> Dict:
    _snake_case   :				Dict														=	np.argmax(p.predictions    ,							axis=1				)

    return {"acc": (preds == p.label_ids).mean()}

  # Initialize our Trainer
  _snake_case   :				str														=	TFTrainer(
      model=_lowerCamelCase    ,							args=_lowerCamelCase    ,							train_dataset=_lowerCamelCase    ,							eval_dataset=_lowerCamelCase    ,							compute_metrics=_lowerCamelCase    ,							)

  # Training
  if training_args.do_train:
    trainer.train()
    trainer.save_model()
    tokenizer.save_pretrained(training_args.output_dir				)

  # Evaluation
  _snake_case   :				Dict														=	{}
  if training_args.do_eval:
    logger.info('*** Evaluate ***'				)
    _snake_case   :				str														=	trainer.evaluate()
    _snake_case   :				Union[str, Any]														=	os.path.join(training_args.output_dir    ,							'eval_results.txt'				)

    with open(_lowerCamelCase    ,							'w'				) as writer:
      logger.info('***** Eval results *****'				)

      for key, value in result.items():
        logger.info(F"""  {key} = {value}"""				)
        writer.write(F"""{key} = {value}\n"""				)

      results.update(_lowerCamelCase				)

  return results


if __name__ == "__main__":
      main()











import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase_						   =    {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Union[str, Any]      					=VOCAB_FILES_NAMES
  a_							:       List[str]      					=PRETRAINED_VOCAB_FILES_MAP
  a_							:       str      					=PRETRAINED_INIT_CONFIGURATION
  a_							:       Optional[Any]      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_							:       List[Any]      					=RealmTokenizer







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Optional[int]=None      ,				UpperCamelCase      :					List[Any]=None      ,				UpperCamelCase      :					Optional[Any]=True      ,				UpperCamelCase      :					Optional[Any]="[UNK]"      ,				UpperCamelCase      :					Any="[SEP]"      ,				UpperCamelCase      :					Optional[Any]="[PAD]"      ,				UpperCamelCase      :					Optional[int]="[CLS]"      ,				UpperCamelCase      :					Optional[Any]="[MASK]"      ,				UpperCamelCase      :					Dict=True      ,				UpperCamelCase      :					Optional[int]=None      ,				**UpperCamelCase      :					Union[str, Any]      ,				):


    '''simple docstring'''





    super().__init__(
        UpperCamelCase      ,				tokenizer_file=UpperCamelCase      ,				do_lower_case=UpperCamelCase      ,				unk_token=UpperCamelCase      ,				sep_token=UpperCamelCase      ,				pad_token=UpperCamelCase      ,				cls_token=UpperCamelCase      ,				mask_token=UpperCamelCase      ,				tokenize_chinese_chars=UpperCamelCase      ,				strip_accents=UpperCamelCase      ,				**UpperCamelCase      ,				)

    _snake_case   :				int														=	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
    if (
        normalizer_state.get('lowercase'      ,				UpperCamelCase							) != do_lower_case
        or normalizer_state.get('strip_accents'      ,				UpperCamelCase							) != strip_accents
        or normalizer_state.get('handle_chinese_chars'      ,				UpperCamelCase							) != tokenize_chinese_chars
    ):
      _snake_case   :				int														=	getattr(UpperCamelCase      ,				normalizer_state.pop('type'							)							)
      _snake_case   :				List[str]														=	do_lower_case
      _snake_case   :				List[Any]														=	strip_accents
      _snake_case   :				Dict														=	tokenize_chinese_chars
      _snake_case   :				Any														=	normalizer_class(**UpperCamelCase							)

    _snake_case   :				Optional[int]														=	do_lower_case







  def UpperCamelCase_ (      self      :					Dict      ,				UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				Tuple														=	PaddingStrategy.MAX_LENGTH

    _snake_case   :				Any														=	text
    _snake_case   :				List[str]														=	kwargs.pop('text_pair'      ,				UpperCamelCase							)
    _snake_case   :				int														=	kwargs.pop('return_tensors'      ,				UpperCamelCase							)

    _snake_case   :				Optional[int]														=	{
        'input_ids': [],
        'attention_mask': [],
        'token_type_ids': [],
    }

    for idx, candidate_text in enumerate(UpperCamelCase							):
      if batch_text_pair is not None:
        _snake_case   :				List[Any]														=	batch_text_pair[idx]
      else:
        _snake_case   :				Optional[Any]														=	None

      _snake_case   :				Optional[int]														=	super().__call__(UpperCamelCase      ,				UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

      _snake_case   :				str														=	encoded_candidates.get('input_ids'							)
      _snake_case   :				Tuple														=	encoded_candidates.get('attention_mask'							)
      _snake_case   :				List[str]														=	encoded_candidates.get('token_type_ids'							)

      if encoded_input_ids is not None:
        output_data["input_ids"].append(UpperCamelCase							)
      if encoded_attention_mask is not None:
        output_data["attention_mask"].append(UpperCamelCase							)
      if encoded_token_type_ids is not None:
        output_data["token_type_ids"].append(UpperCamelCase							)

    _snake_case   :				str														=	{key: item for key, item in output_data.items() if len(UpperCamelCase							) != 0}

    return BatchEncoding(UpperCamelCase      ,				tensor_type=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					Union[str, Any]=None							):


    '''simple docstring'''





    _snake_case   :				Dict														=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					List[int]      ,				UpperCamelCase      :					Optional[List[int]] = None							):


    '''simple docstring'''





    _snake_case   :				int														=	[self.sep_token_id]
    _snake_case   :				Dict														=	[self.cls_token_id]
    if token_ids_a is None:
      return len(cls + token_ids_a + sep							) * [0]
    return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					Optional[str] = None							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	self._tokenizer.model.save(UpperCamelCase      ,				name=UpperCamelCase							)
    return tuple(UpperCamelCase							)




