


"""simple docstring"""


import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch






def      a__ (				__lowercase   )	->	List[Any]:
				_A					  =			os.path.join(args.tf_model_dir     ,			"parameters.json"   )
				_A					  =			json.loads(open(__lowercase   ).read()   )
				if not params:
								raise ValueError(
								    f"""It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file."""   )
				if not args.output.endswith(".pt"   ):
								_A					  =			args.output + ".pt"
				_A					  =			OrderedDict()
				with tf.device("/CPU:0"   ):
								_A					  =			tf.train.load_checkpoint(args.tf_model_dir   )
								_A					  =			reader.get_variable_to_shape_map()
								for key_name in shapes.keys():
												_A					  =			reader.get_tensor(__lowercase   ).astype(np.floataa   )
												if key_name.endswith("/adam_m"   ) or key_name.endswith("/adam_v"   ):
																continue
												if key_name.startswith("pasts/"   ):
																if key_name.startswith("pasts/mlp"   ):
																				_A					  =			int(key_name[9]   )
																elif key_name.startswith("pasts/out"   ):
																				_A					  =			8
																_A					  =			"model.sqout.%d.weight" % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
																_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/moe"   ):
																_A					  =			int(key_name[9:].split("/"   )[0]   )
																if key_name.endswith("/switch_gating/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.router.classifier.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/softmlp/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.soft_bypass_mlp.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/wo/kernel"   ) or key_name.endswith("/wi/kernel"   ):
																				_A					  =			key_name[-9:-7]
																				for i in range(16   ):
																								_A					  =			"model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight" % (player, i, nlayer)
																								_A					  =			(
																								    vnp[i].transpose([1, 0]   ).copy()
																								)  # In Mesh-Tensorflow, it is one array, so it is divided
																								_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/mlp"   ):
																_A					  =			int(key_name[9:].split("/"   )[0]   )
																if key_name.endswith("/p1/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wi.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/p1/bias"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wi.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/p2/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wo.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/p2/bias"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wo.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/ln"   ):
																_A					  =			int(key_name[8:].split("/"   )[0]   )
																if key_name.endswith("/b"   ):
																				_A					  =			"model.blocks.%d.feed_forward.norm.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/g"   ):
																				_A					  =			"model.blocks.%d.feed_forward.norm.weight" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/att"   ):
																_A					  =			int(key_name[9:].split("/"   )[0]   )
																if key_name.endswith("/qkv/kernel"   ):
																				_A					  =			vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
																				_A					  =			state[:, 0, :, :]
																				_A					  =			state[:, 1, :, :]
																				_A					  =			state[:, 2, :, :]
																				_A					  =			(
																				    state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]   )
																				    .transpose([1, 0]   )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			(
																				    state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]   )
																				    .transpose([1, 0]   )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			(
																				    state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]   )
																				    .transpose([1, 0]   )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			"model.blocks.%d.self_attn.self_attn.q_proj.weight" % player
																				_A					  =			torch.tensor(__lowercase   )
																				_A					  =			"model.blocks.%d.self_attn.self_attn.k_proj.weight" % player
																				_A					  =			torch.tensor(__lowercase   )
																				_A					  =			"model.blocks.%d.self_attn.self_attn.v_proj.weight" % player
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/o/kernel"   ):
																				_A					  =			"model.blocks.%d.self_attn.self_attn.out_proj.weight" % player
																				_A					  =			(
																				    vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]   ).transpose([1, 0]   ).copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/an"   ):
																_A					  =			int(key_name[8:].split("/"   )[0]   )
																if key_name.endswith("/b"   ):
																				_A					  =			"model.blocks.%d.self_attn.norm.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/g"   ):
																				_A					  =			"model.blocks.%d.self_attn.norm.weight" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
												elif (
												    key_name.startswith("model/wte"   )
												    or key_name.startswith("model/wpe"   )
												    or key_name.startswith("model/ete"   )
												):
																_A					  =			{"wte": "embed_tokens", "wpe": "position_embeddings", "ete": "extra_position_embeddings"}[
																    key_name[-3:]
																]
																_A					  =			"model.%s.weight" % nlayer
																_A					  =			vnp.copy()  # same in embedded
																_A					  =			torch.tensor(__lowercase   )
																if key_name.startswith("model/wte"   ):
																				_A					  =			"lm_head.weight"
																				_A					  =			vnp.copy()  # same in embedded
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/wob"   ):
																_A					  =			"final_logits_bias"
																_A					  =			vnp.copy()  # same in embedded
																_A					  =			state.reshape((1, -1)   )
																_A					  =			torch.tensor(__lowercase   )
												elif key_name == "model/dense/kernel":
																_A					  =			"model.last_project.weight"
																_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																_A					  =			torch.tensor(__lowercase   )
												elif key_name == "model/dense_1/bias":
																_A					  =			"model.last_project.bias"
																_A					  =			vnp.copy()  # same because it is one dimensional
																_A					  =			torch.tensor(__lowercase   )
				torch.save(__lowercase     ,			args.output   )


if __name__ == "__main__":
		a_       							=    argparse.ArgumentParser(
		    description="model converter.", formatter_class=argparse.ArgumentDefaultsHelpFormatter
		)
		parser.add_argument("--tf_model_dir", metavar="PATH", type=str, required=True, help="import model")
		parser.add_argument("--output", metavar="PATH", type=str, required=True, help="output model")
		a_       							=    parser.parse_args()
		convert_tf_gptsan_to_pt(args)



"""simple docstring"""


import random
import unittest

from torch.utils.data import BatchSampler, DataLoader, IterableDataset

from accelerate import Accelerator
from accelerate.data_loader import (
    BatchSamplerShard,
    DataLoaderDispatcher,
    DataLoaderShard,
    IterableDatasetShard,
    SkipBatchSampler,
    SkipDataLoader,
    skip_first_batches,
)
class snake_case      (  _UpperCamelCase):







		def __init__(					self      :       Optional[int]    ,   a__      :       str=0.0_1    ,   a__      :       str=10_00		)       ->						int:





						'''simple docstring'''







						_A					  =			p_stop
						_A					  =			max_length







		def __iter__(					self      :       Any		)       ->						Optional[Any]:





						'''simple docstring'''







						_A					  =			0
						_A					  =			False
						while not stop and count < self.max_length:
										yield count
										count += 1
										_A					  =			random.random() < self.p_stop





class snake_case      (  unittest.TestCase):







		def  a_ (					self      :       List[Any]    ,   a__      :       Union[str, Any]    ,   a__      :       Union[str, Any]    ,   a__      :       List[str]=False    ,   a__      :       str=True		)       ->						Union[str, Any]:





						'''simple docstring'''







						_A					  =			[
						    BatchSamplerShard(a__    ,   2    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)
						    for i in range(2		)
						]
						_A					  =			[list(a__		) for batch_sampler_shard in batch_sampler_shards]
						if not split_batches:
										self.assertListEqual([len(a__		) for shard in batch_sampler_shards]    ,   [len(a__		) for e in expected]		)
						self.assertListEqual(a__    ,   a__		)







		def  a_ (					self      :       List[Any]		)       ->						str:





						'''simple docstring'''







						_A					  =			BatchSampler(range(24		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
						]
						self.check_batch_sampler_shards(a__    ,   a__		)

						_A					  =			BatchSampler(range(24		)    ,   batch_size=3    ,   drop_last=a__		)
						# Expected shouldn't change
						self.check_batch_sampler_shards(a__    ,   a__		)

						# Check the shards when the dataset is a round multiple of batch size but not total batch size.
						_A					  =			BatchSampler(range(21		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [0, 1, 2]],
						]
						self.check_batch_sampler_shards(a__    ,   a__		)

						_A					  =			BatchSampler(range(21		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__		)

						# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
						# num_processes batch.
						_A					  =			BatchSampler(range(22		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 0, 1]],
						]
						self.check_batch_sampler_shards(a__    ,   a__		)

						_A					  =			BatchSampler(range(22		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__		)

						# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
						# num_processes batch.
						_A					  =			BatchSampler(range(20		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 0]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [1, 2, 3]],
						]
						self.check_batch_sampler_shards(a__    ,   a__		)

						_A					  =			BatchSampler(range(20		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__		)

						# Check the shards when the dataset is very small.
						_A					  =			BatchSampler(range(2		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[[[0, 1, 0]], [[1, 0, 1]]]
						self.check_batch_sampler_shards(a__    ,   a__		)

						_A					  =			BatchSampler(range(2		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[[], []]
						self.check_batch_sampler_shards(a__    ,   a__		)







		def  a_ (					self      :       int		)       ->						int:





						'''simple docstring'''







						_A					  =			BatchSampler(range(24		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)

						_A					  =			BatchSampler(range(24		)    ,   batch_size=4    ,   drop_last=a__		)
						# Expected shouldn't change
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)

						# Check the shards when the dataset is not a round multiple of batch size.
						_A					  =			BatchSampler(range(22		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [0, 1]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)

						_A					  =			BatchSampler(range(22		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)

						# Check the shards when the dataset is not a round multiple of batch size or num_processes.
						_A					  =			BatchSampler(range(21		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 0]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [1, 2]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)

						_A					  =			BatchSampler(range(21		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)

						# Check the shards when the dataset is very small.
						_A					  =			BatchSampler(range(2		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[[[0, 1]], [[0, 1]]]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)

						_A					  =			BatchSampler(range(2		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[[], []]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__		)







		def  a_ (					self      :       List[str]		)       ->						List[str]:





						'''simple docstring'''







						_A					  =			BatchSampler(range(24		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21, 22, 23]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(24		)    ,   batch_size=3    ,   drop_last=a__		)
						# Expected shouldn't change
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						# Check the shards when the dataset is a round multiple of batch size but not total batch size.
						_A					  =			BatchSampler(range(21		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(21		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						# Check the shards when the dataset is not a round multiple of batch size but has a multiple of
						# num_processes batch.
						_A					  =			BatchSampler(range(22		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19, 20]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17], [21]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(22		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						# Check the shards when the dataset is not a round multiple of batch size but and has not a multiple of
						# num_processes batch.
						_A					  =			BatchSampler(range(20		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14], [18, 19]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(20		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1, 2], [6, 7, 8], [12, 13, 14]],
						    [[3, 4, 5], [9, 10, 11], [15, 16, 17]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						# Check the shards when the dataset is very small.
						_A					  =			BatchSampler(range(2		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[[[0, 1]], []]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(2		)    ,   batch_size=3    ,   drop_last=a__		)
						_A					  =			[[], []]
						self.check_batch_sampler_shards(a__    ,   a__    ,   even_batches=a__		)







		def  a_ (					self      :       List[str]		)       ->						str:





						'''simple docstring'''







						_A					  =			BatchSampler(range(24		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19], [22, 23]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(24		)    ,   batch_size=4    ,   drop_last=a__		)
						# Expected shouldn't change
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)

						# Check the shards when the dataset is not a round multiple of batch size.
						_A					  =			BatchSampler(range(22		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20, 21]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(22		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)

						# Check the shards when the dataset is not a round multiple of batch size or num_processes.
						_A					  =			BatchSampler(range(21		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17], [20]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(21		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[
						    [[0, 1], [4, 5], [8, 9], [12, 13], [16, 17]],
						    [[2, 3], [6, 7], [10, 11], [14, 15], [18, 19]],
						]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)

						# Check the shards when the dataset is very small.
						_A					  =			BatchSampler(range(2		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[[[0, 1]], []]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)

						_A					  =			BatchSampler(range(2		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			[[], []]
						self.check_batch_sampler_shards(a__    ,   a__    ,   split_batches=a__    ,   even_batches=a__		)







		def  a_ (					self      :       Union[str, Any]		)       ->						str:





						'''simple docstring'''







						_A					  =			[[0, 1, 2], [3, 4], [5, 6, 7, 8], [9, 10, 11], [12, 13]]
						_A					  =			[BatchSamplerShard(a__    ,   2    ,   a__    ,   even_batches=a__		) for i in range(2		)]

						self.assertEqual(len(batch_sampler_shards[0]		)    ,   3		)
						self.assertEqual(len(batch_sampler_shards[1]		)    ,   2		)

						self.assertListEqual(list(batch_sampler_shards[0]		)    ,   [[0, 1, 2], [5, 6, 7, 8], [12, 13]]		)
						self.assertListEqual(list(batch_sampler_shards[1]		)    ,   [[3, 4], [9, 10, 11]]		)







		def  a_ (					self      :       Optional[int]    ,   a__      :       Optional[int]    ,   a__      :       Tuple    ,   a__      :       Optional[int]    ,   a__      :       Union[str, Any]=False    ,   a__      :       int=2    ,   a__      :       List[Any]=False		)       ->						str:





						'''simple docstring'''







						random.seed(a__		)
						_A					  =			list(a__		)

						_A					  =			[
						    IterableDatasetShard(
						        a__    ,   batch_size=a__    ,   drop_last=a__    ,   num_processes=a__    ,   process_index=a__    ,   split_batches=a__    ,   )
						    for i in range(a__		)
						]
						_A					  =			[]
						for iterable_dataset_shard in iterable_dataset_shards:
										# Since our random iterable dataset will be... random... we need to use a seed to get reproducible results.
										random.seed(a__		)
										iterable_dataset_lists.append(list(a__		)		)

						_A					  =			batch_size // num_processes if split_batches else batch_size
						# All iterable dataset shard should have the same length, a round multiple of shard_batch_size
						_A					  =			iterable_dataset_lists[0]
						for l in iterable_dataset_lists[1:]:
										self.assertEqual(len(a__		)    ,   len(a__		)		)
										self.assertTrue(len(a__		) % shard_batch_size == 0		)

						_A					  =			[]
						for idx in range(0    ,   len(a__		)    ,   a__		):
										for l in iterable_dataset_lists:
														observed += l[idx : idx + shard_batch_size]

						if not drop_last:
										while len(a__		) < len(a__		):
														reference += reference
						self.assertListEqual(a__    ,   reference[: len(a__		)]		)







		def  a_ (					self      :       List[str]		)       ->						List[Any]:





						'''simple docstring'''







						_A					  =			42
						_A					  =			RandomIterableDataset()

						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)
						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)
						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)
						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)

						# Edge case with a very small dataset
						_A					  =			RandomIterableDataset(max_length=2		)

						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)
						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)
						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)
						self.check_iterable_dataset_shards(a__    ,   a__    ,   batch_size=4    ,   drop_last=a__    ,   split_batches=a__		)







		def  a_ (					self      :       List[str]		)       ->						Dict:





						'''simple docstring'''







						_A					  =			BatchSampler(range(16		)    ,   batch_size=4    ,   drop_last=a__		)
						_A					  =			SkipBatchSampler(a__    ,   2		)
						self.assertListEqual(list(a__		)    ,   [[8, 9, 10, 11], [12, 13, 14, 15]]		)







		def  a_ (					self      :       int		)       ->						Union[str, Any]:





						'''simple docstring'''







						_A					  =			SkipDataLoader(list(range(16		)		)    ,   batch_size=4    ,   skip_batches=2		)
						self.assertListEqual([t.tolist() for t in dataloader]    ,   [[8, 9, 10, 11], [12, 13, 14, 15]]		)







		def  a_ (					self      :       int		)       ->						Optional[int]:





						'''simple docstring'''







						_A					  =			DataLoader(list(range(16		)		)    ,   batch_size=4		)
						_A					  =			skip_first_batches(a__    ,   num_batches=2		)
						self.assertListEqual([t.tolist() for t in new_dataloader]    ,   [[8, 9, 10, 11], [12, 13, 14, 15]]		)







		def  a_ (					self      :       Optional[Any]		)       ->						Optional[int]:





						'''simple docstring'''







						_A					  =			DataLoaderShard(list(range(16		)		)    ,   batch_size=4		)
						for idx, _ in enumerate(a__		):
										self.assertEqual(dataloader.end_of_dataloader    ,   idx == 3		)

						# Test it also works on the second iteration
						for idx, _ in enumerate(a__		):
										self.assertEqual(dataloader.end_of_dataloader    ,   idx == 3		)







		def  a_ (					self      :       int		)       ->						int:





						'''simple docstring'''







						Accelerator()
						_A					  =			DataLoaderDispatcher(range(16		)    ,   batch_size=4		)
						for idx, _ in enumerate(a__		):
										self.assertEqual(dataloader.end_of_dataloader    ,   idx == 3		)

						# Test it also works on the second iteration
						for idx, _ in enumerate(a__		):
										self.assertEqual(dataloader.end_of_dataloader    ,   idx == 3		)