
import argparse
import logging
import os
import re

import tensorflow as tf

from transformers import (
    AutoConfig,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    PushToHubCallback,
    TFAutoModelForMaskedLM,
    create_optimizer,
)


lowercase__	:Optional[Any]										=					logging.getLogger(__name__)

lowercase__	:List[str]										=					tf.data.AUTOTUNE



def  UpperCamelCase					(							):
					'''simple docstring'''
					lowercase					=		argparse.ArgumentParser(description='''Train a masked language model on TPU.'''	)
					parser.add_argument(
					    '''--pretrained_model_config'''				,    type=lowerCAmelCase__				,    default='''roberta-base'''				,    help='''The model config to use. Note that we don\'t copy the model\'s weights, only the config!'''				,    )
					parser.add_argument(
					    '''--tokenizer'''				,    type=lowerCAmelCase__				,    default='''unigram-tokenizer-wikitext'''				,    help='''The name of the tokenizer to load. We use the pretrained tokenizer to initialize the model\'s vocab size.'''				,    )

					parser.add_argument(
					    '''--per_replica_batch_size'''				,    type=lowerCAmelCase__				,    default=8				,    help='''Batch size per TPU core.'''				,    )

					parser.add_argument(
					    '''--no_tpu'''				,    action='''store_true'''				,    help='''If set, run on CPU and don\'t try to initialize a TPU. Useful for debugging on non-TPU instances.'''				,    )

					parser.add_argument(
					    '''--tpu_name'''				,    type=lowerCAmelCase__				,    help='''Name of TPU resource to initialize. Should be blank on Colab, and \'local\' on TPU VMs.'''				,    default='''local'''				,    )

					parser.add_argument(
					    '''--tpu_zone'''				,    type=lowerCAmelCase__				,    help='''Google cloud zone that TPU resource is located in. Only used for non-Colab TPU nodes.'''				,    )

					parser.add_argument(
					    '''--gcp_project'''				,    type=lowerCAmelCase__				,    help='''Google cloud project name. Only used for non-Colab TPU nodes.'''	)

					parser.add_argument(
					    '''--bfloat16'''				,    action='''store_true'''				,    help='''Use mixed-precision bfloat16 for training. This is the recommended lower-precision format for TPU.'''				,    )

					parser.add_argument(
					    '''--train_dataset'''				,    type=lowerCAmelCase__				,    help='''Path to training dataset to load. If the path begins with `gs://`'''
					    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''				,    )

					parser.add_argument(
					    '''--shuffle_buffer_size'''				,    type=lowerCAmelCase__				,    default=2**18				,    help='''Size of the shuffle buffer (in samples)'''				,    )

					parser.add_argument(
					    '''--eval_dataset'''				,    type=lowerCAmelCase__				,    help='''Path to evaluation dataset to load. If the path begins with `gs://`'''
					    ''' then the dataset will be loaded from a Google Cloud Storage bucket.'''				,    )

					parser.add_argument(
					    '''--num_epochs'''				,    type=lowerCAmelCase__				,    default=1				,    help='''Number of epochs to train for.'''				,    )

					parser.add_argument(
					    '''--learning_rate'''				,    type=lowerCAmelCase__				,    default=1E-4				,    help='''Learning rate to use for training.'''				,    )

					parser.add_argument(
					    '''--weight_decay_rate'''				,    type=lowerCAmelCase__				,    default=1E-3				,    help='''Weight decay rate to use for training.'''				,    )

					parser.add_argument(
					    '''--max_length'''				,    type=lowerCAmelCase__				,    default=512				,    help='''Maximum length of tokenized sequences. Should match the setting used in prepare_tfrecord_shards.py'''				,    )

					parser.add_argument(
					    '''--mlm_probability'''				,    type=lowerCAmelCase__				,    default=0.15				,    help='''Fraction of tokens to mask during training.'''				,    )

					parser.add_argument('''--output_dir'''				,    type=lowerCAmelCase__				,    required=lowerCAmelCase__				,    help='''Path to save model checkpoints to.'''	)
					parser.add_argument('''--hub_model_id'''				,    type=lowerCAmelCase__				,    help='''Model ID to upload to on the Hugging Face Hub.'''	)

					lowercase					=		parser.parse_args()
					return args



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					try:
										if args.tpu_name:
															lowercase					=		tf.distribute.cluster_resolver.TPUClusterResolver(
															    args.tpu_name				,    zone=args.tpu_zone				,    project=args.gcp_project	)
										else:
															lowercase					=		tf.distribute.cluster_resolver.TPUClusterResolver()
					except ValueError:
										raise RuntimeError(
										    '''Couldn\'t connect to TPU! Most likely you need to specify --tpu_name, --tpu_zone, or '''
										    '''--gcp_project. When running on a TPU VM, use --tpu_name local.'''	)

					tf.config.experimental_connect_to_cluster(lowerCAmelCase__	)
					tf.tpu.experimental.initialize_tpu_system(lowerCAmelCase__	)

					return tpu



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					lowercase					=		0
					for file in file_list:
										lowercase					=		file.split('''/'''	)[-1]
										lowercase					=		re.search(R'''-\d+-(\d+)\.tfrecord'''				,    lowerCAmelCase__	).group(1	)
										lowercase					=		int(lowerCAmelCase__	)
										num_samples += sample_count

					return num_samples



def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__=None	):
					'''simple docstring'''
					lowercase					=		count_samples(lowerCAmelCase__	)
					lowercase					=		tf.data.Dataset.from_tensor_slices(lowerCAmelCase__	)
					if shuffle:
										lowercase					=		dataset.shuffle(len(lowerCAmelCase__	)	)
					lowercase					=		tf.data.TFRecordDataset(lowerCAmelCase__				,    num_parallel_reads=lowerCAmelCase__	)
					# TF can't infer the total sample count because it doesn't read all the records yet, so we assert it here
					lowercase					=		dataset.apply(tf.data.experimental.assert_cardinality(lowerCAmelCase__	)	)
					lowercase					=		dataset.map(lowerCAmelCase__				,    num_parallel_calls=lowerCAmelCase__	)
					if shuffle:
										assert shuffle_buffer_size is not None
										lowercase					=		dataset.shuffle(args.shuffle_buffer_size	)
					lowercase					=		dataset.batch(lowerCAmelCase__				,    drop_remainder=lowerCAmelCase__	)
					lowercase					=		dataset.map(lowerCAmelCase__				,    num_parallel_calls=lowerCAmelCase__	)
					lowercase					=		dataset.prefetch(lowerCAmelCase__	)
					return dataset



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					if not args.no_tpu:
										lowercase					=		initialize_tpu(lowerCAmelCase__	)
										lowercase					=		tf.distribute.TPUStrategy(lowerCAmelCase__	)
					else:
										lowercase					=		tf.distribute.OneDeviceStrategy(device='''/gpu:0'''	)

					if args.bfloataa:
										tf.keras.mixed_precision.set_global_policy('''mixed_bfloat16'''	)

					lowercase					=		AutoTokenizer.from_pretrained(args.tokenizer	)
					lowercase					=		AutoConfig.from_pretrained(args.pretrained_model_config	)
					lowercase					=		tokenizer.vocab_size

					lowercase					=		tf.io.gfile.glob(os.path.join(args.train_dataset				,    '''*.tfrecord'''	)	)
					if not training_records:
										raise ValueError(f'No .tfrecord files found in {args.train_dataset}.'	)
					lowercase					=		tf.io.gfile.glob(os.path.join(args.eval_dataset				,    '''*.tfrecord'''	)	)
					if not eval_records:
										raise ValueError(f'No .tfrecord files found in {args.eval_dataset}.'	)

					lowercase					=		count_samples(lowerCAmelCase__	)

					lowercase					=		num_train_samples // (args.per_replica_batch_size * strategy.num_replicas_in_sync)
					lowercase					=		steps_per_epoch * args.num_epochs

					with strategy.scope():
										lowercase					=		TFAutoModelForMaskedLM.from_config(lowerCAmelCase__	)
										model(model.dummy_inputs	)  # Pass some dummy inputs through the model to ensure all the weights are built
										lowercase			,		lowercase					=		create_optimizer(
										    num_train_steps=lowerCAmelCase__				,    num_warmup_steps=total_train_steps // 20				,    init_lr=args.learning_rate				,    weight_decay_rate=args.weight_decay_rate				,    )

										# Transformers models compute the right loss for their task by default when labels are passed, and will
										# use this for training unless you specify your own loss function in compile().
										model.compile(optimizer=lowerCAmelCase__				,    metrics=['''accuracy''']	)

					def decode_fn(lowerCAmelCase__	):
										lowercase					=		{
										    '''input_ids''': tf.io.FixedLenFeature(dtype=tf.intaa				,    shape=(args.max_length,)	),
										    '''attention_mask''': tf.io.FixedLenFeature(dtype=tf.intaa				,    shape=(args.max_length,)	),
										}
										return tf.io.parse_single_example(lowerCAmelCase__				,    lowerCAmelCase__	)

					# Many of the data collators in Transformers are TF-compilable when return_tensors == "tf", so we can
					# use their methods in our data pipeline.
					lowercase					=		DataCollatorForLanguageModeling(
					    tokenizer=lowerCAmelCase__				,    mlm_probability=args.mlm_probability				,    mlm=lowerCAmelCase__				,    return_tensors='''tf'''	)

					def mask_with_collator(lowerCAmelCase__	):
										# TF really needs an isin() function
										lowercase					=		(
										    ~tf.cast(batch['''attention_mask''']				,    tf.bool	)
										    | (batch['''input_ids'''] == tokenizer.cls_token_id)
										    | (batch['''input_ids'''] == tokenizer.sep_token_id)
										)
										lowercase			,		lowercase					=		data_collator.tf_mask_tokens(
										    batch['''input_ids''']				,    vocab_size=len(lowerCAmelCase__	)				,    mask_token_id=tokenizer.mask_token_id				,    special_tokens_mask=lowerCAmelCase__				,    )
										return batch

					lowercase					=		args.per_replica_batch_size * strategy.num_replicas_in_sync

					lowercase					=		prepare_dataset(
					    lowerCAmelCase__				,    decode_fn=lowerCAmelCase__				,    mask_fn=lowerCAmelCase__				,    batch_size=lowerCAmelCase__				,    shuffle=lowerCAmelCase__				,    shuffle_buffer_size=args.shuffle_buffer_size				,    )

					lowercase					=		prepare_dataset(
					    lowerCAmelCase__				,    decode_fn=lowerCAmelCase__				,    mask_fn=lowerCAmelCase__				,    batch_size=lowerCAmelCase__				,    shuffle=lowerCAmelCase__				,    )

					lowercase					=		[]
					if args.hub_model_id:
										callbacks.append(
										    PushToHubCallback(output_dir=args.output_dir				,    hub_model_id=args.hub_model_id				,    tokenizer=lowerCAmelCase__	)	)

					model.fit(
					    lowerCAmelCase__				,    validation_data=lowerCAmelCase__				,    epochs=args.num_epochs				,    callbacks=lowerCAmelCase__				,    )

					model.save_pretrained(args.output_dir	)


if __name__ == "__main__":
							lowercase__	:Dict										=					parse_args()
							main(args)






import asyncio
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
from distutils.util import strtobool
from functools import partial
from pathlib import Path
from typing import List, Union
from unittest import mock

import torch

from ..state import AcceleratorState, PartialState
from ..utils import (
    gather,
    is_bnb_available,
    is_comet_ml_available,
    is_datasets_available,
    is_deepspeed_available,
    is_mps_available,
    is_safetensors_available,
    is_tensorboard_available,
    is_torch_version,
    is_tpu_available,
    is_transformers_available,
    is_wandb_available,
    is_xpu_available,
)



def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__=False	):
					'''simple docstring'''
					try:
										lowercase					=		os.environ[key]
					except KeyError:
										# KEY isn't set, default to `default`.
										lowercase					=		default
					else:
										# KEY is set, convert it to True or False.
										try:
															lowercase					=		strtobool(lowerCAmelCase__	)
										except ValueError:
															# More values are supported, but let's keep the message simple.
															raise ValueError(f'If set, {key} must be yes or no.'	)
					return _value


lowercase__	:Dict										=					parse_flag_from_env("RUN_SLOW", default=False)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skip('''Test was skipped'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(_run_slow_tests				,    '''test is slow'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(not torch.cuda.is_available()				,    '''test requires only a CPU'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(torch.cuda.is_available()				,    '''test requires a GPU'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_xpu_available()				,    '''test requires a XPU'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_mps_available()				,    '''test requires a `mps` backend support in `torch`'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(
					    is_transformers_available() and is_datasets_available()				,    '''test requires the Hugging Face suite'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_bnb_available()				,    '''test requires the bitsandbytes library'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_tpu_available()				,    '''test requires TPU'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(torch.cuda.device_count() == 1				,    '''test requires a GPU'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(torch.xpu.device_count() == 1				,    '''test requires a XPU'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(torch.cuda.device_count() > 1				,    '''test requires multiple GPUs'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(torch.xpu.device_count() > 1				,    '''test requires multiple XPUs'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_safetensors_available()				,    '''test requires safetensors'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_deepspeed_available()				,    '''test requires DeepSpeed'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_torch_version('''>='''				,    '''1.12.0'''	)				,    '''test requires torch version >= 1.12.0'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__=None				,    lowerCAmelCase__=None	):
					'''simple docstring'''
					if test_case is None:
										return partial(lowerCAmelCase__				,    version=lowerCAmelCase__	)
					return unittest.skipUnless(is_torch_version('''>='''				,    lowerCAmelCase__	)				,    f'test requires torch version >= {version}'	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_tensorboard_available()				,    '''test requires Tensorboard'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_wandb_available()				,    '''test requires wandb'''	)(lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(is_comet_ml_available()				,    '''test requires comet_ml'''	)(lowerCAmelCase__	)


lowercase__	:Dict										=					(
    any([is_wandb_available(), is_tensorboard_available()]) and not is_comet_ml_available()
)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					return unittest.skipUnless(
					    _atleast_one_tracker_available				,    '''test requires at least one tracker to be available and for `comet_ml` to not be installed'''				,    )(lowerCAmelCase__	)







class        lowercase	(      unittest.TestCase	):
							lowercase_ : int       =True




							@classmethod
							def 				A__   (       cls):
												lowercase					=		tempfile.mkdtemp()




							@classmethod
							def 				A__   (       cls):
												if os.path.exists(cls.tmpdir):
																	shutil.rmtree(cls.tmpdir)





							def 				A__   (       self):
												if self.clear_on_setup:
																	for path in Path(self.tmpdir).glob('''**/*'''):
																						if path.is_file():
																											path.unlink()
																						elif path.is_dir():
																											shutil.rmtree(A__)







class        lowercase	(      unittest.TestCase	):




							def 				A__   (       self):
												super().tearDown()
												# Reset the state of the AcceleratorState singleton.
												AcceleratorState._reset_state()
												PartialState._reset_state()







class        lowercase	(      unittest.TestCase	):




							def 				A__   (       self     ,A__):
												lowercase					=		mocks if isinstance(A__     ,(tuple, list)) else [mocks]
												for m in self.mocks:
																	m.start()
																	self.addCleanup(m.stop)



def  UpperCamelCase					(							lowerCAmelCase__	):
					'''simple docstring'''
					lowercase					=		AcceleratorState()
					lowercase					=		tensor[None].clone().to(state.device	)
					lowercase					=		gather(lowerCAmelCase__	).cpu()
					lowercase					=		tensor[0].cpu()
					for i in range(tensors.shape[0]	):
										if not torch.equal(tensors[i]				,    lowerCAmelCase__	):
															return False
					return True







class        lowercase	:




							def __init__(       self     ,A__     ,A__     ,A__):
												lowercase					=		returncode
												lowercase					=		stdout
												lowercase					=		stderr



async def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__	):
					'''simple docstring'''
					while True:
										lowercase					=		await stream.readline()
										if line:
															callback(lowerCAmelCase__	)
										else:
															break



async def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__=None				,    lowerCAmelCase__=None				,    lowerCAmelCase__=None				,    lowerCAmelCase__=False				,    lowerCAmelCase__=False	):
					'''simple docstring'''
					if echo:
										print('''\nRunning: '''				,    ''' '''.join(lowerCAmelCase__	)	)

					lowercase					=		await asyncio.create_subprocess_exec(
					    cmd[0]				,    *cmd[1:]				,    stdin=lowerCAmelCase__				,    stdout=asyncio.subprocess.PIPE				,    stderr=asyncio.subprocess.PIPE				,    env=lowerCAmelCase__				,    )

					# note: there is a warning for a possible deadlock when using `wait` with huge amounts of data in the pipe
					# https://docs.python.org/3/library/asyncio-subprocess.html#asyncio.asyncio.subprocess.Process.wait
					#
					# If it starts hanging, will need to switch to the following code. The problem is that no data
					# will be seen until it's done and if it hangs for example there will be no debug info.
					# out, err = await p.communicate()
					# return _RunOutput(p.returncode, out, err)

					lowercase					=		[]
					lowercase					=		[]

					def tee(lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__				,    lowerCAmelCase__=""	):
										lowercase					=		line.decode('''utf-8'''	).rstrip()
										sink.append(lowerCAmelCase__	)
										if not quiet:
															print(lowerCAmelCase__				,    lowerCAmelCase__				,    file=lowerCAmelCase__	)

    # XXX: the timeout doesn't seem to make any difference here
					await asyncio.wait(
					    [
					        asyncio.create_task(_read_stream(p.stdout				,    lambda lowerCAmelCase__	: tee(lowerCAmelCase__				,    lowerCAmelCase__				,    sys.stdout				,    label='''stdout:'''	)	)	),
					        asyncio.create_task(_read_stream(p.stderr				,    lambda lowerCAmelCase__	: tee(lowerCAmelCase__				,    lowerCAmelCase__				,    sys.stderr				,    label='''stderr:'''	)	)	),
					    ]				,    timeout=lowerCAmelCase__				,    )
					return _RunOutput(await p.wait()				,    lowerCAmelCase__				,    lowerCAmelCase__	)



def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__=None				,    lowerCAmelCase__=None				,    lowerCAmelCase__=180				,    lowerCAmelCase__=False				,    lowerCAmelCase__=True	):
					'''simple docstring'''
					lowercase					=		asyncio.get_event_loop()
					lowercase					=		loop.run_until_complete(
					    _stream_subprocess(lowerCAmelCase__				,    env=lowerCAmelCase__				,    stdin=lowerCAmelCase__				,    timeout=lowerCAmelCase__				,    quiet=lowerCAmelCase__				,    echo=lowerCAmelCase__	)	)

					lowercase					=		''' '''.join(lowerCAmelCase__	)
					if result.returncode > 0:
										lowercase					=		'''\n'''.join(result.stderr	)
										raise RuntimeError(
										    f'\'{cmd_str}\' failed with returncode {result.returncode}\n\n'
										    f'The combined stderr from workers follows:\n{stderr}'	)

					return result







class        lowercase	(      SCREAMING_SNAKE_CASE__	):
							pass



def  UpperCamelCase					(							lowerCAmelCase__				,    lowerCAmelCase__=False	):
					'''simple docstring'''
					try:
										lowercase					=		subprocess.check_output(lowerCAmelCase__				,    stderr=subprocess.STDOUT	)
										if return_stdout:
															if hasattr(lowerCAmelCase__				,    '''decode'''	):
																				lowercase					=		output.decode('''utf-8'''	)
															return output
					except subprocess.CalledProcessError as e:
										raise SubprocessCallException(
										    f'Command `{" ".join(lowerCAmelCase__	)}` failed with the following error:\n\n{e.output.decode()}'	) from e




