



import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
  from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
  a_			=					True
  from torch.cuda.amp import autocast


a_			=					logging.getLogger(__name__)



@dataclass
class  __lowerCAmelCase     :
 lowerCAmelCase__       					=  field(
     metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}   )
 lowerCAmelCase__       					=  field(
     default=lowerCAmelCase__  ,	metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}  ,	)
 lowerCAmelCase__       					=  field(
     default=lowerCAmelCase__  ,	metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}   )
 lowerCAmelCase__       					=  field(
     default=lowerCAmelCase__  ,	metadata={"""help""": """Whether to log verbose messages or not."""}  ,	)
 lowerCAmelCase__       					=  field(
     default=2.0  ,	metadata={"""help""": """Maximum temperature for gumbel softmax."""}   )
 lowerCAmelCase__       					=  field(
     default=0.5  ,	metadata={"""help""": """Minimum temperature for gumbel softmax."""}   )
 lowerCAmelCase__       					=  field(
     default=0.99_99_95  ,	metadata={"""help""": """Decay of gumbel temperature during training."""}   )





def 		a__			(		_UpperCamelCase			: ModelArguments ,_UpperCamelCase			: TrainingArguments							):
   logging.basicConfig(
       format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''' ,datefmt='''%m/%d/%Y %H:%M:%S''' ,handlers=[logging.StreamHandler(sys.stdout							)] ,)
   __lowerCamelCase													=    logging.WARNING
   if model_args.verbose_logging:
      __lowerCamelCase													=    logging.DEBUG
   elif trainer_utils.is_main_process(training_args.local_rank							):
      __lowerCamelCase													=    logging.INFO
   logger.setLevel(_UpperCamelCase							)



@dataclass
class  __lowerCAmelCase     :
 lowerCAmelCase__       					=  field(
     default=lowerCAmelCase__  ,	metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}   )
 lowerCAmelCase__       					=  field(
     default=lowerCAmelCase__  ,	metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}   )
 lowerCAmelCase__       					=  field(
     default="""train"""  ,	metadata={
         """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
     }  ,	)
 lowerCAmelCase__       					=  field(
     default="""validation"""  ,	metadata={
         """help""": (
             """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
         )
     }  ,	)
 lowerCAmelCase__       					=  field(
     default="""file"""  ,	metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}  ,	)
 lowerCAmelCase__       					=  field(
     default=lowerCAmelCase__  ,	metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}   )
 lowerCAmelCase__       					=  field(
     default=1  ,	metadata={
         """help""": """The percentage of the train set used as validation set in case there's no validation split"""
     }  ,	)
 lowerCAmelCase__       					=  field(
     default=lowerCAmelCase__  ,	metadata={"""help""": """The number of processes to use for the preprocessing."""}  ,	)
 lowerCAmelCase__       					=  field(
     default=20.0  ,	metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}   )



@dataclass
class  __lowerCAmelCase     :
 lowerCAmelCase__       					=  42
 lowerCAmelCase__       					=  42
 lowerCAmelCase__       					=  "longest"
 lowerCAmelCase__       					=  None
 lowerCAmelCase__       					=  None




 def __call__(						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    # reformat list to dict and set to pytorch format
    __lowerCamelCase													=    self.feature_extractor.pad(
        __UpperCAmelCase	,				max_length=self.max_length	,				padding=self.padding	,				pad_to_multiple_of=self.pad_to_multiple_of	,				return_tensors='''pt'''	,				)
    __lowerCamelCase													=    self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]				)

    __lowerCamelCase													=    batch['''input_values'''].shape[0]

    # make sure that no loss is computed on padded inputs
    if batch["attention_mask"] is not None:
       # compute real output lengths according to convolution formula
       __lowerCamelCase													=    self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1				)				).to(
           torch.long				)

       __lowerCamelCase													=    torch.zeros(
           (batch_size, mask_indices_seq_length)	,				dtype=torch.long	,				device=batch['''input_values'''].device				)

       # these two operations makes sure that all values
       # before the output lengths indices are attended to
       __lowerCamelCase													=    1
       __lowerCamelCase													=    attention_mask.flip([-1]				).cumsum(-1				).flip([-1]				).bool()

    # sample randomly masked indices
    __lowerCamelCase													=    _compute_mask_indices(
        (batch_size, mask_indices_seq_length)	,				self.model.config.mask_time_prob	,				self.model.config.mask_time_length	,				attention_mask=__UpperCAmelCase	,				min_masks=2	,				)

    return batch



class  __lowerCAmelCase     (	lowerCAmelCase__   ):




 def __init__(						self	,				*__UpperCAmelCase	,				__UpperCAmelCase=1	,				__UpperCAmelCase=0	,				__UpperCAmelCase=1.0	,				**__UpperCAmelCase				):



    '''simple docstring'''
    super().__init__(*__UpperCAmelCase	,				**__UpperCAmelCase				)
    __lowerCamelCase													=    0
    __lowerCamelCase													=    max_gumbel_temp
    __lowerCamelCase													=    min_gumbel_temp
    __lowerCamelCase													=    gumbel_temp_decay




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase				):



    '''simple docstring'''
    model.train()
    __lowerCamelCase													=    self._prepare_inputs(__UpperCAmelCase				)

    if self.use_amp:
       with autocast():
          __lowerCamelCase													=    self.compute_loss(__UpperCAmelCase	,				__UpperCAmelCase				)
    else:
       __lowerCamelCase													=    self.compute_loss(__UpperCAmelCase	,				__UpperCAmelCase				)

    if self.args.n_gpu > 1 or self.deepspeed:
       if model.module.config.ctc_loss_reduction == "mean":
          __lowerCamelCase													=    loss.mean()
       elif model.module.config.ctc_loss_reduction == "sum":
          __lowerCamelCase													=    loss.sum() / (inputs['''mask_time_indices''']).sum()
       else:
          raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""				)

    if self.args.gradient_accumulation_steps > 1:
       __lowerCamelCase													=    loss / self.args.gradient_accumulation_steps

    if self.use_amp:
       self.scaler.scale(__UpperCAmelCase				).backward()
    elif self.use_apex:
       with amp.scale_loss(__UpperCAmelCase	,				self.optimizer				) as scaled_loss:
          scaled_loss.backward()
    elif self.deepspeed:
       self.deepspeed.backward(__UpperCAmelCase				)
    else:
       loss.backward()

    self.num_update_step += 1
    # make sure gumbel softmax temperature is decayed
    if self.args.n_gpu > 1 or self.deepspeed:
       model.module.set_gumbel_temperature(
           max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step	,				self.min_gumbel_temp				)				)
    else:
       model.set_gumbel_temperature(
           max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step	,				self.min_gumbel_temp				)				)

    return loss.detach()





def 		a__			(		):
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   __lowerCamelCase													=    HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)

   __lowerCamelCase							,__lowerCamelCase							,__lowerCamelCase													=    parser.parse_args_into_dataclasses()
   configure_logger(_UpperCamelCase ,_UpperCamelCase							)

   # Downloading and loading a dataset from the hub.
   __lowerCamelCase													=    load_dataset(data_args.dataset_name ,data_args.dataset_config_name ,cache_dir=model_args.cache_dir							)

   if "validation" not in datasets.keys():
      # make sure only "validation" and "train" keys remain"
      __lowerCamelCase													=    DatasetDict()
      __lowerCamelCase													=    load_dataset(
          data_args.dataset_name ,data_args.dataset_config_name ,split=F"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]""" ,cache_dir=model_args.cache_dir ,)
      __lowerCamelCase													=    load_dataset(
          data_args.dataset_name ,data_args.dataset_config_name ,split=F"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]""" ,cache_dir=model_args.cache_dir ,)
   else:
      # make sure only "validation" and "train" keys remain"
      __lowerCamelCase													=    DatasetDict()
      __lowerCamelCase													=    load_dataset(
          data_args.dataset_name ,data_args.dataset_config_name ,split='''validation''' ,cache_dir=model_args.cache_dir ,)
      __lowerCamelCase													=    load_dataset(
          data_args.dataset_name ,data_args.dataset_config_name ,split=F"""{data_args.train_split_name}""" ,cache_dir=model_args.cache_dir ,)

   # only normalized-inputs-training is supported
   __lowerCamelCase													=    WavaVecaFeatureExtractor.from_pretrained(
       model_args.model_name_or_path ,cache_dir=model_args.cache_dir ,do_normalize=_UpperCamelCase							)

   def prepare_dataset(_UpperCamelCase			: Optional[Any]							):
      # check that all files have the correct sampling rate
      __lowerCamelCase							,__lowerCamelCase													=    librosa.load(batch[data_args.speech_file_column] ,sr=feature_extractor.sampling_rate							)
      return batch

   # load audio files into numpy arrays
   __lowerCamelCase													=    datasets.map(
       _UpperCamelCase ,num_proc=data_args.preprocessing_num_workers ,remove_columns=datasets['''train'''].column_names							)

   # filter audio files that are too long
   __lowerCamelCase													=    vectorized_datasets.filter(
       lambda _UpperCamelCase							: len(data['''speech''']							) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate							)							)

   def normalize(_UpperCamelCase			: List[Any]							):
      return feature_extractor(batch['''speech'''] ,sampling_rate=feature_extractor.sampling_rate							)

   # normalize and transform to `BatchFeatures`
   __lowerCamelCase													=    vectorized_datasets.map(
       _UpperCamelCase ,batched=_UpperCamelCase ,num_proc=data_args.preprocessing_num_workers ,load_from_cache_file=not data_args.overwrite_cache ,remove_columns=vectorized_datasets['''train'''].column_names ,)

   # pretraining is only supported for "newer" stable layer norm architecture
   # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
   __lowerCamelCase													=    WavaVecaConfig.from_pretrained(
       model_args.model_name_or_path ,cache_dir=model_args.cache_dir ,gradient_checkpointing=training_args.gradient_checkpointing ,)

   if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
      raise ValueError(
          '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
          ''' ``config.feat_extract_norm=\'layer\''''							)

   __lowerCamelCase													=    WavaVecaForPreTraining(_UpperCamelCase							)

   __lowerCamelCase													=    DataCollatorForWavaVecaPretraining(model=_UpperCamelCase ,feature_extractor=_UpperCamelCase							)

   __lowerCamelCase													=    WavaVecaPreTrainer(
       model=_UpperCamelCase ,data_collator=_UpperCamelCase ,args=_UpperCamelCase ,train_dataset=vectorized_datasets['''train'''] ,eval_dataset=vectorized_datasets['''validation'''] ,tokenizer=_UpperCamelCase ,max_gumbel_temp=model_args.max_gumbel_temperature ,min_gumbel_temp=model_args.min_gumbel_temperature ,gumbel_temp_decay=model_args.gumbel_temperature_decay ,)
   trainer.train()


if __name__ == "__main__":
  main()











def 		a__			(		_UpperCamelCase			: str ,_UpperCamelCase			: str = " "							):
   __lowerCamelCase													=    []

   __lowerCamelCase													=    0
   for index, char in enumerate(_UpperCamelCase							):
      if char == separator:
         split_words.append(string[last_index:index]							)
         __lowerCamelCase													=    index + 1
      elif index + 1 == len(_UpperCamelCase							):
         split_words.append(string[last_index : index + 1]							)
   return split_words


if __name__ == "__main__":
  from doctest import testmod

  testmod()






