


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
   from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('1.6'):
   __UpperCamelCase				:  Dict      					=							True
   from torch.cuda.amp import autocast


__UpperCamelCase				:  List[Any]      					=							logging.getLogger(__name__)





@dataclass
class  lowerCAmelCase__:






    '''simple docstring'''





    A_  :     List[str]			= field(
        metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}						)
    A_  :     List[str]			= field(
        default=snake_case__    ,    metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}    ,    )
    A_  :     List[str]			= field(
        default=snake_case__    ,    metadata={'help': 'Whether to freeze the feature extractor layers of the model.'}						)
    A_  :     int			= field(
        default=snake_case__    ,    metadata={'help': 'Whether to log verbose messages or not.'}    ,    )
    A_  :     Optional[Any]			= field(
        default=2.0    ,    metadata={'help': 'Maximum temperature for gumbel softmax.'}						)
    A_  :     str			= field(
        default=0.5    ,    metadata={'help': 'Minimum temperature for gumbel softmax.'}						)
    A_  :     Optional[Any]			= field(
        default=0.9_9_9_9_9_5    ,    metadata={'help': 'Decay of gumbel temperature during training.'}						)





def      snake_case_						( __lowercase			,       __lowercase       ):
   logging.basicConfig(
       format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''			,       datefmt='''%m/%d/%Y %H:%M:%S'''			,       handlers=[logging.StreamHandler(sys.stdout       )]			,       )
   UpperCAmelCase_							:      Any						=   logging.WARNING
   if model_args.verbose_logging:
      UpperCAmelCase_							:      str						=   logging.DEBUG
   elif trainer_utils.is_main_process(training_args.local_rank       ):
      UpperCAmelCase_							:      List[str]						=   logging.INFO
   logger.setLevel(__SCREAMING_SNAKE_CASE       )





@dataclass
class  lowerCAmelCase__:






    '''simple docstring'''





    A_  :     str			= field(
        default=snake_case__    ,    metadata={'help': 'The name of the dataset to use (via the datasets library).'}						)
    A_  :     Tuple			= field(
        default=snake_case__    ,    metadata={'help': 'The configuration name of the dataset to use (via the datasets library).'}						)
    A_  :     List[str]			= field(
        default='train'    ,    metadata={
            'help': 'The name of the training data set split to use (via the datasets library). Defaults to \'train\''
        }    ,    )
    A_  :     List[Any]			= field(
        default='validation'    ,    metadata={
            'help': (
                'The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''
            )
        }    ,    )
    A_  :     Union[str, Any]			= field(
        default='file'    ,    metadata={'help': 'Column in the dataset that contains speech file path. Defaults to \'file\''}    ,    )
    A_  :     Dict			= field(
        default=snake_case__    ,    metadata={'help': 'Overwrite the cached preprocessed datasets or not.'}						)
    A_  :     Optional[int]			= field(
        default=1    ,    metadata={
            'help': 'The percentage of the train set used as validation set in case there\'s no validation split'
        }    ,    )
    A_  :     List[str]			= field(
        default=snake_case__    ,    metadata={'help': 'The number of processes to use for the preprocessing.'}    ,    )
    A_  :     str			= field(
        default=2_0.0    ,    metadata={'help': 'Filter audio files that are longer than `max_duration_in_seconds` seconds'}						)





@dataclass
class  lowerCAmelCase__:






    '''simple docstring'''





    A_  :     List[Any]			= 4_2
    A_  :     Any			= 4_2
    A_  :     List[Any]			= 'longest'
    A_  :     List[Any]			= None
    A_  :     List[str]			= None





    def __call__(	self				:   List[Any]       ,   __snake_case				:   Dict	):





       '''simple docstring'''
       # reformat list to dict and set to pytorch format
       UpperCAmelCase_							:      List[Any]						=   self.feature_extractor.pad(
           lowercase_       ,   max_length=self.max_length       ,   padding=self.padding       ,   pad_to_multiple_of=self.pad_to_multiple_of       ,   return_tensors='''pt'''       ,   )
       UpperCAmelCase_							:      Optional[int]						=   self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]	)

       UpperCAmelCase_							:      Tuple						=   batch["input_values"].shape[0]

       # make sure that no loss is computed on padded inputs
       if batch["attention_mask"] is not None:
          # compute real output lengths according to convolution formula
          UpperCAmelCase_							:      Tuple						=   self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1	)	).to(
              torch.long	)

          UpperCAmelCase_							:      Union[str, Any]						=   torch.zeros(
              (batch_size, mask_indices_seq_length)       ,   dtype=torch.long       ,   device=batch['''input_values'''].device	)

          # these two operations makes sure that all values
          # before the output lengths indices are attended to
          UpperCAmelCase_							:      Optional[Any]						=   1
          UpperCAmelCase_							:      Dict						=   attention_mask.flip([-1]	).cumsum(-1	).flip([-1]	).bool()

       # sample randomly masked indices
       UpperCAmelCase_							:      str						=   _compute_mask_indices(
           (batch_size, mask_indices_seq_length)       ,   self.model.config.mask_time_prob       ,   self.model.config.mask_time_length       ,   attention_mask=lowercase_       ,   min_masks=2       ,   )

       return batch





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    def __init__(	self				:   str       ,   *__snake_case				:   Any       ,   __snake_case				:   Dict=1       ,   __snake_case				:   Dict=0       ,   __snake_case				:   str=1.0       ,   **__snake_case				:   Tuple	):





       '''simple docstring'''
       super().__init__(*lowercase_       ,   **lowercase_	)
       UpperCAmelCase_							:      Union[str, Any]						=   0
       UpperCAmelCase_							:      int						=   max_gumbel_temp
       UpperCAmelCase_							:      Union[str, Any]						=   min_gumbel_temp
       UpperCAmelCase_							:      List[str]						=   gumbel_temp_decay





    def 					_lowerCamelCase (	self				:   Tuple       ,   __snake_case				:   Optional[int]       ,   __snake_case				:   Any	):





       '''simple docstring'''
       model.train()
       UpperCAmelCase_							:      Dict						=   self._prepare_inputs(lowercase_	)

       if self.use_amp:
          with autocast():
             UpperCAmelCase_							:      Dict						=   self.compute_loss(lowercase_       ,   lowercase_	)
       else:
          UpperCAmelCase_							:      Tuple						=   self.compute_loss(lowercase_       ,   lowercase_	)

       if self.args.n_gpu > 1 or self.deepspeed:
          if model.module.config.ctc_loss_reduction == "mean":
             UpperCAmelCase_							:      Any						=   loss.mean()
          elif model.module.config.ctc_loss_reduction == "sum":
             UpperCAmelCase_							:      List[Any]						=   loss.sum() / (inputs["mask_time_indices"]).sum()
          else:
             raise ValueError(f'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''	)

       if self.args.gradient_accumulation_steps > 1:
          UpperCAmelCase_							:      int						=   loss / self.args.gradient_accumulation_steps

       if self.use_amp:
          self.scaler.scale(lowercase_	).backward()
       elif self.use_apex:
          with amp.scale_loss(lowercase_       ,   self.optimizer	) as scaled_loss:
             scaled_loss.backward()
       elif self.deepspeed:
          self.deepspeed.backward(lowercase_	)
       else:
          loss.backward()

       self.num_update_step += 1
       # make sure gumbel softmax temperature is decayed
       if self.args.n_gpu > 1 or self.deepspeed:
          model.module.set_gumbel_temperature(
              max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,   self.min_gumbel_temp	)	)
       else:
          model.set_gumbel_temperature(
              max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step       ,   self.min_gumbel_temp	)	)

       return loss.detach()





def      snake_case_						( ):
   UpperCAmelCase_							:      str						=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

   UpperCAmelCase_							:      str						=   parser.parse_args_into_dataclasses()
   configure_logger(__SCREAMING_SNAKE_CASE			,       __SCREAMING_SNAKE_CASE       )

   # Downloading and loading a dataset from the hub.
   UpperCAmelCase_							:      Optional[Any]						=   load_dataset(data_args.dataset_name			,       data_args.dataset_config_name			,       cache_dir=model_args.cache_dir       )

   if "validation" not in datasets.keys():
      # make sure only "validation" and "train" keys remain"
      UpperCAmelCase_							:      Optional[int]						=   DatasetDict()
      UpperCAmelCase_							:      Optional[Any]						=   load_dataset(
          data_args.dataset_name			,       data_args.dataset_config_name			,       split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''			,       cache_dir=model_args.cache_dir			,       )
      UpperCAmelCase_							:      Tuple						=   load_dataset(
          data_args.dataset_name			,       data_args.dataset_config_name			,       split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''			,       cache_dir=model_args.cache_dir			,       )
   else:
      # make sure only "validation" and "train" keys remain"
      UpperCAmelCase_							:      Tuple						=   DatasetDict()
      UpperCAmelCase_							:      Dict						=   load_dataset(
          data_args.dataset_name			,       data_args.dataset_config_name			,       split='''validation'''			,       cache_dir=model_args.cache_dir			,       )
      UpperCAmelCase_							:      Union[str, Any]						=   load_dataset(
          data_args.dataset_name			,       data_args.dataset_config_name			,       split=F'''{data_args.train_split_name}'''			,       cache_dir=model_args.cache_dir			,       )

   # only normalized-inputs-training is supported
   UpperCAmelCase_							:      List[str]						=   WavaVecaFeatureExtractor.from_pretrained(
       model_args.model_name_or_path			,       cache_dir=model_args.cache_dir			,       do_normalize=__SCREAMING_SNAKE_CASE       )

   def prepare_dataset(__lowercase       ):
      # check that all files have the correct sampling rate
      UpperCAmelCase_							:      Optional[Any]						=   librosa.load(batch[data_args.speech_file_column]			,       sr=feature_extractor.sampling_rate       )
      return batch

   # load audio files into numpy arrays
   UpperCAmelCase_							:      Optional[int]						=   datasets.map(
       __SCREAMING_SNAKE_CASE			,       num_proc=data_args.preprocessing_num_workers			,       remove_columns=datasets['''train'''].column_names       )

   # filter audio files that are too long
   UpperCAmelCase_							:      Any						=   vectorized_datasets.filter(
       lambda __lowercase       : len(data['''speech''']       ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate       )       )

   def normalize(__lowercase       ):
      return feature_extractor(batch['''speech''']			,       sampling_rate=feature_extractor.sampling_rate       )

   # normalize and transform to `BatchFeatures`
   UpperCAmelCase_							:      Optional[Any]						=   vectorized_datasets.map(
       __SCREAMING_SNAKE_CASE			,       batched=__SCREAMING_SNAKE_CASE			,       num_proc=data_args.preprocessing_num_workers			,       load_from_cache_file=not data_args.overwrite_cache			,       remove_columns=vectorized_datasets['''train'''].column_names			,       )

   # pretraining is only supported for "newer" stable layer norm architecture
   # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
   UpperCAmelCase_							:      Optional[int]						=   WavaVecaConfig.from_pretrained(
       model_args.model_name_or_path			,       cache_dir=model_args.cache_dir			,       gradient_checkpointing=training_args.gradient_checkpointing			,       )

   if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
      raise ValueError(
          '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
          ''' ``config.feat_extract_norm=\'layer\''''       )

   UpperCAmelCase_							:      int						=   WavaVecaForPreTraining(__SCREAMING_SNAKE_CASE       )

   UpperCAmelCase_							:      Dict						=   DataCollatorForWavaVecaPretraining(model=__SCREAMING_SNAKE_CASE			,       feature_extractor=__SCREAMING_SNAKE_CASE       )

   UpperCAmelCase_							:      str						=   WavaVecaPreTrainer(
       model=__SCREAMING_SNAKE_CASE			,       data_collator=__SCREAMING_SNAKE_CASE			,       args=__SCREAMING_SNAKE_CASE			,       train_dataset=vectorized_datasets['''train''']			,       eval_dataset=vectorized_datasets['''validation''']			,       tokenizer=__SCREAMING_SNAKE_CASE			,       max_gumbel_temp=model_args.max_gumbel_temperature			,       min_gumbel_temp=model_args.min_gumbel_temperature			,       gumbel_temp_decay=model_args.gumbel_temperature_decay			,       )
   trainer.train()


if __name__ == "__main__":
   main()



import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_ta import TaTokenizer
else:
       __UpperCamelCase				:  Optional[int]      					=							None


__UpperCamelCase				:  Union[str, Any]      					=							logging.get_logger(__name__)

__UpperCamelCase				:  List[Any]      					=							{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

__UpperCamelCase				:  Optional[int]      					=							{
    'vocab_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/spiece.model',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/spiece.model',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/spiece.model',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/spiece.model',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        't5-small': 'https://huggingface.co/t5-small/resolve/main/tokenizer.json',
        't5-base': 'https://huggingface.co/t5-base/resolve/main/tokenizer.json',
        't5-large': 'https://huggingface.co/t5-large/resolve/main/tokenizer.json',
        't5-3b': 'https://huggingface.co/t5-3b/resolve/main/tokenizer.json',
        't5-11b': 'https://huggingface.co/t5-11b/resolve/main/tokenizer.json',
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__UpperCamelCase				:  Optional[Any]      					=							{
    't5-small': 512,
    't5-base': 512,
    't5-large': 512,
    't5-3b': 512,
    't5-11b': 512,
}





class  lowerCAmelCase__(    snake_case__						):






    '''simple docstring'''





    A_  :     Optional[int]			= VOCAB_FILES_NAMES
    A_  :     List[Any]			= PRETRAINED_VOCAB_FILES_MAP
    A_  :     int			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    A_  :     str			= ['input_ids', 'attention_mask']
    A_  :     int			= TaTokenizer

    A_  :     List[int]			= []





    def __init__(	self				:   Union[str, Any]       ,   __snake_case				:   Tuple=None       ,   __snake_case				:   List[Any]=None       ,   __snake_case				:   int="</s>"       ,   __snake_case				:   List[Any]="<unk>"       ,   __snake_case				:   Dict="<pad>"       ,   __snake_case				:   Tuple=100       ,   __snake_case				:   int=None       ,   **__snake_case				:   Any       ,   ):





       '''simple docstring'''
       # Add extra_ids to the special token list
       if extra_ids > 0 and additional_special_tokens is None:
          UpperCAmelCase_							:      Optional[int]						=   [f'''<extra_id_{i}>''' for i in range(__snake_case	)]
       elif extra_ids > 0 and additional_special_tokens is not None:
          # Check that we have the right number of extra special tokens
          UpperCAmelCase_							:      Any						=   len(set(filter(lambda __snake_case	: bool('''extra_id_''' in str(__snake_case	)	)       ,   __snake_case	)	)	)
          if extra_tokens != extra_ids:
             raise ValueError(
                 f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
                 ''' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'''
                 ''' tokens'''	)

       super().__init__(
           __snake_case       ,   tokenizer_file=__snake_case       ,   eos_token=__snake_case       ,   unk_token=__snake_case       ,   pad_token=__snake_case       ,   extra_ids=__snake_case       ,   additional_special_tokens=__snake_case       ,   **__snake_case       ,   )

       UpperCAmelCase_							:      str						=   vocab_file
       UpperCAmelCase_							:      List[str]						=   False if not self.vocab_file else True
       UpperCAmelCase_							:      Union[str, Any]						=   extra_ids





    @staticmethod
    def 					_lowerCamelCase (	__snake_case				:   Dict       ,   __snake_case				:   List[str]       ,   __snake_case				:   Tuple	):





       '''simple docstring'''
       if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
          UpperCAmelCase_							:      str						=   TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
          if init_max_model_length is not None and init_max_model_length != max_model_length:
             return init_max_model_length
          elif init_max_model_length is None:
             warnings.warn(
                 '''This tokenizer was incorrectly instantiated with a model max length of'''
                 f''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
                 ''' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'''
                 ''' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'''
                 f''' {pretrained_model_name_or_path} automatically truncating your input to'''
                 f''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
                 f''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
                 ''' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'''
                 ''' instantiate this tokenizer with `model_max_length` set to your preferred value.'''       ,   __snake_case       ,   )

       return max_model_length





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   str       ,   __snake_case				:   Optional[str] = None	):





       '''simple docstring'''
       if not self.can_save_slow_tokenizer:
          raise ValueError(
              '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
              '''tokenizer.'''	)

       if not os.path.isdir(__snake_case	):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
          return
       UpperCAmelCase_							:      str						=   os.path.join(
           __snake_case       ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

       if os.path.abspath(self.vocab_file	) != os.path.abspath(__snake_case	):
          copyfile(self.vocab_file       ,   __snake_case	)
          logger.info(f'''Copy vocab file to {out_vocab_file}'''	)

       return (out_vocab_file,)





    def 					_lowerCamelCase (	self				:   List[str]       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      Optional[Any]						=   token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
          return self.prefix_tokens + token_ids_a
       else:
          UpperCAmelCase_							:      int						=   token_ids_a + [self.eos_token_id]
          return self.prefix_tokens + token_ids_a + token_ids_a





    def 					_lowerCamelCase (	self				:   Dict       ,   __snake_case				:   List[int]       ,   __snake_case				:   Optional[List[int]] = None	):





       '''simple docstring'''
       UpperCAmelCase_							:      int						=   [self.eos_token_id]

       if token_ids_a is None:
          return len(token_ids_a + eos	) * [0]
       return len(token_ids_a + eos + token_ids_a + eos	) * [0]





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       return list(
           set(filter(lambda __snake_case	: bool(re.search(R'''<extra_id_\d+>'''       ,   __snake_case	)	) is not None       ,   self.additional_special_tokens	)	)	)





    def 					_lowerCamelCase (	self				:   Tuple	):





       '''simple docstring'''
       return [self.convert_tokens_to_ids(__snake_case	) for token in self.get_sentinel_tokens()]