




from dataclasses import dataclass
from typing import Optional

import numpy as np
import torch
import torch.nn as nn

from ..utils import BaseOutput, is_torch_version, randn_tensor
from .attention_processor import SpatialNorm
from .unet_ad_blocks import UNetMidBlockaD, get_down_block, get_up_block




@dataclass
class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_ :							torch.FloatTensor




class 					snake_case__							(nn.Module     ):
				"""simple docstring"""



				def __init__(   self       :  Union[str, Any]      ,   __lowerCamelCase       :  List[str]=3      ,   __lowerCamelCase       :  Union[str, Any]=3      ,   __lowerCamelCase       :  str=("DownEncoderBlock2D",)      ,   __lowerCamelCase       :  int=(64,)      ,   __lowerCamelCase       :  str=2      ,   __lowerCamelCase       :  List[Any]=32      ,   __lowerCamelCase       :  Optional[int]="silu"      ,   __lowerCamelCase       :  int=True      ,   )      ->  Any:
							super().__init__()
							a					=    layers_per_block

							a					=    torch.nn.Convad(
							    __lowerCamelCase      ,   block_out_channels[0]      ,   kernel_size=3      ,   stride=1      ,   padding=1      ,   )

							a					=    None
							a					=    nn.ModuleList([]							)

							# down
							a					=    block_out_channels[0]
							for i, down_block_type in enumerate(__lowerCamelCase							):
										a					=    output_channel
										a					=    block_out_channels[i]
										a					=    i == len(__lowerCamelCase							) - 1

										a					=    get_down_block(
										    __lowerCamelCase      ,   num_layers=self.layers_per_block      ,   in_channels=__lowerCamelCase      ,   out_channels=__lowerCamelCase      ,   add_downsample=not is_final_block      ,   resnet_eps=1e-6      ,   downsample_padding=0      ,   resnet_act_fn=__lowerCamelCase      ,   resnet_groups=__lowerCamelCase      ,   attention_head_dim=__lowerCamelCase      ,   temb_channels=__lowerCamelCase      ,   )
										self.down_blocks.append(__lowerCamelCase							)

							# mid
							a					=    UNetMidBlockaD(
							    in_channels=block_out_channels[-1]      ,   resnet_eps=1e-6      ,   resnet_act_fn=__lowerCamelCase      ,   output_scale_factor=1      ,   resnet_time_scale_shift="default"      ,   attention_head_dim=block_out_channels[-1]      ,   resnet_groups=__lowerCamelCase      ,   temb_channels=__lowerCamelCase      ,   )

							# out
							a					=    nn.GroupNorm(num_channels=block_out_channels[-1]      ,   num_groups=__lowerCamelCase      ,   eps=1e-6							)
							a					=    nn.SiLU()

							a					=    2 * out_channels if double_z else out_channels
							a					=    nn.Convad(block_out_channels[-1]      ,   __lowerCamelCase      ,   3      ,   padding=1							)

							a					=    False





				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  Tuple							)      ->  List[str]:
							a					=    x
							a					=    self.conv_in(__lowerCamelCase							)

							if self.training and self.gradient_checkpointing:

										def create_custom_forward(__lowerCamelCase       :  str							):
													def custom_forward(*__lowerCamelCase       :  List[Any]							):
																return module(*__lowerCamelCase							)

													return custom_forward

										# down
										if is_torch_version(">="      ,   "1.11.0"							):
													for down_block in self.down_blocks:
																a					=    torch.utils.checkpoint.checkpoint(
																    create_custom_forward(__lowerCamelCase							)      ,   __lowerCamelCase      ,   use_reentrant=__lowerCamelCase							)
													# middle
													a					=    torch.utils.checkpoint.checkpoint(
													    create_custom_forward(self.mid_block							)      ,   __lowerCamelCase      ,   use_reentrant=__lowerCamelCase							)
										else:
													for down_block in self.down_blocks:
																a					=    torch.utils.checkpoint.checkpoint(create_custom_forward(__lowerCamelCase							)      ,   __lowerCamelCase							)
													# middle
													a					=    torch.utils.checkpoint.checkpoint(create_custom_forward(self.mid_block							)      ,   __lowerCamelCase							)

							else:
										# down
										for down_block in self.down_blocks:
													a					=    down_block(__lowerCamelCase							)

										# middle
										a					=    self.mid_block(__lowerCamelCase							)

							# post-process
							a					=    self.conv_norm_out(__lowerCamelCase							)
							a					=    self.conv_act(__lowerCamelCase							)
							a					=    self.conv_out(__lowerCamelCase							)

							return sample




class 					snake_case__							(nn.Module     ):
				"""simple docstring"""



				def __init__(   self       :  int      ,   __lowerCamelCase       :  int=3      ,   __lowerCamelCase       :  Tuple=3      ,   __lowerCamelCase       :  str=("UpDecoderBlock2D",)      ,   __lowerCamelCase       :  Optional[Any]=(64,)      ,   __lowerCamelCase       :  Any=2      ,   __lowerCamelCase       :  str=32      ,   __lowerCamelCase       :  str="silu"      ,   __lowerCamelCase       :  str="group"      ,   )      ->  Tuple:
							super().__init__()
							a					=    layers_per_block

							a					=    nn.Convad(
							    __lowerCamelCase      ,   block_out_channels[-1]      ,   kernel_size=3      ,   stride=1      ,   padding=1      ,   )

							a					=    None
							a					=    nn.ModuleList([]							)

							a					=    in_channels if norm_type == "spatial" else None

							# mid
							a					=    UNetMidBlockaD(
							    in_channels=block_out_channels[-1]      ,   resnet_eps=1e-6      ,   resnet_act_fn=__lowerCamelCase      ,   output_scale_factor=1      ,   resnet_time_scale_shift="default" if norm_type == "group" else norm_type      ,   attention_head_dim=block_out_channels[-1]      ,   resnet_groups=__lowerCamelCase      ,   temb_channels=__lowerCamelCase      ,   )

							# up
							a					=    list(reversed(__lowerCamelCase							)							)
							a					=    reversed_block_out_channels[0]
							for i, up_block_type in enumerate(__lowerCamelCase							):
										a					=    output_channel
										a					=    reversed_block_out_channels[i]

										a					=    i == len(__lowerCamelCase							) - 1

										a					=    get_up_block(
										    __lowerCamelCase      ,   num_layers=self.layers_per_block + 1      ,   in_channels=__lowerCamelCase      ,   out_channels=__lowerCamelCase      ,   prev_output_channel=__lowerCamelCase      ,   add_upsample=not is_final_block      ,   resnet_eps=1e-6      ,   resnet_act_fn=__lowerCamelCase      ,   resnet_groups=__lowerCamelCase      ,   attention_head_dim=__lowerCamelCase      ,   temb_channels=__lowerCamelCase      ,   resnet_time_scale_shift=__lowerCamelCase      ,   )
										self.up_blocks.append(__lowerCamelCase							)
										a					=    output_channel

							# out
							if norm_type == "spatial":
										a					=    SpatialNorm(block_out_channels[0]      ,   __lowerCamelCase							)
							else:
										a					=    nn.GroupNorm(num_channels=block_out_channels[0]      ,   num_groups=__lowerCamelCase      ,   eps=1e-6							)
							a					=    nn.SiLU()
							a					=    nn.Convad(block_out_channels[0]      ,   __lowerCamelCase      ,   3      ,   padding=1							)

							a					=    False





				def 							__UpperCAmelCase							(   self       :  int      ,   __lowerCamelCase       :  List[str]      ,   __lowerCamelCase       :  str=None							)      ->  Union[str, Any]:
							a					=    z
							a					=    self.conv_in(__lowerCamelCase							)

							a					=    next(iter(self.up_blocks.parameters()							)							).dtype
							if self.training and self.gradient_checkpointing:

										def create_custom_forward(__lowerCamelCase       :  Tuple							):
													def custom_forward(*__lowerCamelCase       :  List[str]							):
																return module(*__lowerCamelCase							)

													return custom_forward

										if is_torch_version(">="      ,   "1.11.0"							):
													# middle
													a					=    torch.utils.checkpoint.checkpoint(
													    create_custom_forward(self.mid_block							)      ,   __lowerCamelCase      ,   __lowerCamelCase      ,   use_reentrant=__lowerCamelCase							)
													a					=    sample.to(__lowerCamelCase							)

													# up
													for up_block in self.up_blocks:
																a					=    torch.utils.checkpoint.checkpoint(
																    create_custom_forward(__lowerCamelCase							)      ,   __lowerCamelCase      ,   __lowerCamelCase      ,   use_reentrant=__lowerCamelCase							)
										else:
													# middle
													a					=    torch.utils.checkpoint.checkpoint(
													    create_custom_forward(self.mid_block							)      ,   __lowerCamelCase      ,   __lowerCamelCase							)
													a					=    sample.to(__lowerCamelCase							)

													# up
													for up_block in self.up_blocks:
																a					=    torch.utils.checkpoint.checkpoint(create_custom_forward(__lowerCamelCase							)      ,   __lowerCamelCase      ,   __lowerCamelCase							)
							else:
										# middle
										a					=    self.mid_block(__lowerCamelCase      ,   __lowerCamelCase							)
										a					=    sample.to(__lowerCamelCase							)

										# up
										for up_block in self.up_blocks:
													a					=    up_block(__lowerCamelCase      ,   __lowerCamelCase							)

        # post-process
							if latent_embeds is None:
										a					=    self.conv_norm_out(__lowerCamelCase							)
							else:
										a					=    self.conv_norm_out(__lowerCamelCase      ,   __lowerCamelCase							)
							a					=    self.conv_act(__lowerCamelCase							)
							a					=    self.conv_out(__lowerCamelCase							)

							return sample




class 					snake_case__							(nn.Module     ):
				"""simple docstring"""



				def __init__(   self       :  Tuple      ,   __lowerCamelCase       :  int      ,   __lowerCamelCase       :  List[Any]      ,   __lowerCamelCase       :  Tuple      ,   __lowerCamelCase       :  Union[str, Any]=None      ,   __lowerCamelCase       :  int="random"      ,   __lowerCamelCase       :  Optional[Any]=False      ,   __lowerCamelCase       :  Optional[int]=True							)      ->  Tuple:
							super().__init__()
							a					=    n_e
							a					=    vq_embed_dim
							a					=    beta
							a					=    legacy

							a					=    nn.Embedding(self.n_e      ,   self.vq_embed_dim							)
							self.embedding.weight.data.uniform_(-1.0 / self.n_e      ,   1.0 / self.n_e							)

							a					=    remap
							if self.remap is not None:
										self.register_buffer("used"      ,   torch.tensor(np.load(self.remap							)							)							)
										a					=    self.used.shape[0]
										a					=    unknown_index  # "random" or "extra" or integer
										if self.unknown_index == "extra":
													a					=    self.re_embed
													a					=    self.re_embed + 1
										print(
										    f"""Remapping {self.n_e} indices to {self.re_embed} indices. """
										    f"""Using {self.unknown_index} for unknown indices."""							)
							else:
										a					=    n_e

							a					=    sane_index_shape





				def 							__UpperCAmelCase							(   self       :  Dict      ,   __lowerCamelCase       :  int							)      ->  List[str]:
							a					=    inds.shape
							assert len(__lowerCamelCase							) > 1
							a					=    inds.reshape(ishape[0]      ,   -1							)
							a					=    self.used.to(__lowerCamelCase							)
							a					=    (inds[:, :, None] == used[None, None, ...]).long()
							a					=    match.argmax(-1							)
							a					=    match.sum(2							) < 1
							if self.unknown_index == "random":
										a					=    torch.randint(0      ,   self.re_embed      ,   size=new[unknown].shape							).to(device=new.device							)
							else:
										a					=    self.unknown_index
							return new.reshape(__lowerCamelCase							)





				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  List[str]							)      ->  Any:
							a					=    inds.shape
							assert len(__lowerCamelCase							) > 1
							a					=    inds.reshape(ishape[0]      ,   -1							)
							a					=    self.used.to(__lowerCamelCase							)
							if self.re_embed > self.used.shape[0]:  # extra token
										a					=    0  # simply set to zero
							a					=    torch.gather(used[None, :][inds.shape[0] * [0], :]      ,   1      ,   __lowerCamelCase							)
							return back.reshape(__lowerCamelCase							)





				def 							__UpperCAmelCase							(   self       :  Tuple      ,   __lowerCamelCase       :  Dict							)      ->  int:
							# reshape z -> (batch, height, width, channel) and flatten
							a					=    z.permute(0      ,   2      ,   3      ,   1							).contiguous()
							a					=    z.view(-1      ,   self.vq_embed_dim							)

							# distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z
							a					=    torch.argmin(torch.cdist(__lowerCamelCase      ,   self.embedding.weight							)      ,   dim=1							)

							a					=    self.embedding(__lowerCamelCase							).view(z.shape							)
							a					=    None
							a					=    None

							# compute loss for embedding
							if not self.legacy:
										a					=    self.beta * torch.mean((z_q.detach() - z) ** 2							) + torch.mean((z_q - z.detach()) ** 2							)
							else:
										a					=    torch.mean((z_q.detach() - z) ** 2							) + self.beta * torch.mean((z_q - z.detach()) ** 2							)

							# preserve gradients
							a					=    z + (z_q - z).detach()

							# reshape back to match original input shape
							a					=    z_q.permute(0      ,   3      ,   1      ,   2							).contiguous()

							if self.remap is not None:
										a					=    min_encoding_indices.reshape(z.shape[0]      ,   -1							)  # add batch axis
										a					=    self.remap_to_used(__lowerCamelCase							)
										a					=    min_encoding_indices.reshape(-1      ,   1							)  # flatten

							if self.sane_index_shape:
										a					=    min_encoding_indices.reshape(z_q.shape[0]      ,   z_q.shape[2]      ,   z_q.shape[3]							)

							return z_q, loss, (perplexity, min_encodings, min_encoding_indices)





				def 							__UpperCAmelCase							(   self       :  Tuple      ,   __lowerCamelCase       :  int      ,   __lowerCamelCase       :  Tuple							)      ->  int:
							# shape specifying (batch, height, width, channel)
							if self.remap is not None:
										a					=    indices.reshape(shape[0]      ,   -1							)  # add batch axis
										a					=    self.unmap_to_all(__lowerCamelCase							)
										a					=    indices.reshape(-1							)  # flatten again

							# get quantized latent vectors
							a					=    self.embedding(__lowerCamelCase							)

							if shape is not None:
										a					=    z_q.view(__lowerCamelCase							)
										# reshape back to match original input shape
										a					=    z_q.permute(0      ,   3      ,   1      ,   2							).contiguous()

							return z_q




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				def __init__(   self       :  List[str]      ,   __lowerCamelCase       :  Tuple      ,   __lowerCamelCase       :  List[str]=False							)      ->  Union[str, Any]:
							a					=    parameters
							a					,	a					=    torch.chunk(__lowerCamelCase      ,   2      ,   dim=1							)
							a					=    torch.clamp(self.logvar      ,   -30.0      ,   20.0							)
							a					=    deterministic
							a					=    torch.exp(0.5 * self.logvar							)
							a					=    torch.exp(self.logvar							)
							if self.deterministic:
										a					=    a					=    torch.zeros_like(
										    self.mean      ,   device=self.parameters.device      ,   dtype=self.parameters.dtype							)





				def 							__UpperCAmelCase							(   self       :  int      ,   __lowerCamelCase       :  Optional[torch.Generator] = None							)      ->  torch.FloatTensor:
							# make sure sample is on the same device as the parameters and has same dtype
							a					=    randn_tensor(
							    self.mean.shape      ,   generator=__lowerCamelCase      ,   device=self.parameters.device      ,   dtype=self.parameters.dtype							)
							a					=    self.mean + self.std * sample
							return x





				def 							__UpperCAmelCase							(   self       :  List[Any]      ,   __lowerCamelCase       :  Union[str, Any]=None							)      ->  str:
							if self.deterministic:
										return torch.Tensor([0.0]							)
							else:
										if other is None:
													return 0.5 * torch.sum(torch.pow(self.mean      ,   2							) + self.var - 1.0 - self.logvar      ,   dim=[1, 2, 3]							)
										else:
													return 0.5 * torch.sum(
													    torch.pow(self.mean - other.mean      ,   2							) / other.var
													    + self.var / other.var
													    - 1.0
													    - self.logvar
													    + other.logvar      ,   dim=[1, 2, 3]      ,   )





				def 							__UpperCAmelCase							(   self       :  Optional[int]      ,   __lowerCamelCase       :  Union[str, Any]      ,   __lowerCamelCase       :  Union[str, Any]=[1, 2, 3]							)      ->  List[str]:
							if self.deterministic:
										return torch.Tensor([0.0]							)
							a					=    np.log(2.0 * np.pi							)
							return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean      ,   2							) / self.var      ,   dim=__lowerCamelCase							)





				def 							__UpperCAmelCase							(   self       :  Optional[Any]							)      ->  List[str]:
							return self.mean












import argparse
import logging
import os

import datasets
import tensorflow as tf

from transformers import AutoTokenizer


__lowerCAmelCase	:       List[Any]     =	logging.getLogger(__name__)




def     __magic_name__       (				):






			'''simple docstring'''







			a					=    argparse.ArgumentParser(
			    description="Prepare TFRecord shards from pre-tokenized samples of the wikitext dataset."     )
			parser.add_argument(
			    "--dataset_name",						type=A,						default="wikitext",						help="Name of the training. Explore datasets at: hf.co/datasets.",						)
			parser.add_argument(
			    "--dataset_config",						type=A,						default="wikitext-103-raw-v1",						help="Configuration name of the dataset."     )
			parser.add_argument(
			    "--tokenizer_name_or_path",						type=A,						default="sayakpaul/unigram-tokenizer-wikitext",						help="Tokenizer identifier. Can be a local filepath or a Hub identifier.",						)
			parser.add_argument(
			    "--shard_size",						type=A,						default=1000,						help="Number of entries to go in a single shard.",						)
			parser.add_argument("--split",						type=A,						default="train",						choices=["train", "test", "validation"]     )
			parser.add_argument(
			    "--limit",						default=A,						type=A,						help="Limit the number of shards (used for debugging).",						)
			parser.add_argument(
			    "--max_length",						type=A,						default=512,						help="Maximum sequence length. For training on TPUs, it helps to have a maximum"
			    " sequence length that is a multiple of 8.",						)
			parser.add_argument(
			    "--output_dir",						default="tf-tpu",						type=A,						help="Output directory where the TFRecord shards will be saved. If the"
			    " path is appended with `gs://` ('gs://tf-tpu', for example) then the TFRecord"
			    " shards will be directly saved to a Google Cloud Storage bucket.",						)

			a					=    parser.parse_args()
			return args




def     __magic_name__       (				A	:      List[str]     ):






			'''simple docstring'''







			def fn(A	:      Tuple     ):
						return tokenizer(examples["text"]     )

			return fn




def     __magic_name__       (				A	:      Any     ):






			'''simple docstring'''







			a					=    []
			for i in range(len(tokenized_data["input_ids"]     )     ):
						a					=    {
						    "input_ids": tf.train.Feature(intaa_list=tf.train.IntaaList(value=tokenized_data["input_ids"][i]     )     ),
						    "attention_mask": tf.train.Feature(
						        intaa_list=tf.train.IntaaList(value=tokenized_data["attention_mask"][i]     )     ),
						}
						a					=    tf.train.Features(feature=A     )
						a					=    tf.train.Example(features=A     )
						a					=    example.SerializeToString()
						records.append(A     )
			return records




def     __magic_name__       (				A	:      Union[str, Any]     ):






			'''simple docstring'''







			a					=    datasets.load_dataset(args.dataset_name,						args.dataset_config,						split=args.split     )

			if args.limit is not None:
						a					=    min(len(A     ),						args.limit     )
						a					=    dataset.select(range(A     )     )
						print(F"""Limiting the dataset to {args.limit} entries."""     )

			a					=    AutoTokenizer.from_pretrained(args.tokenizer_name_or_path     )

			# Handle output directory creation.
			# For serializing into a Google Cloud Storage Bucket, one needs to first
			# create a bucket.
			if "gs" not in args.output_dir:
						if not os.path.exists(args.output_dir     ):
									os.makedirs(args.output_dir     )
						a					=    os.path.join(args.output_dir,						args.split     )
						if not os.path.exists(A     ):
									os.makedirs(A     )
			else:
						a					=    os.path.join(args.output_dir,						args.split     )

			# Tokenize the whole dataset at once.
			a					=    tokenize_function(A     )
			a					=    dataset.map(A,						batched=A,						num_proc=4,						remove_columns=["text"]     )

			# We need to concatenate all our texts together, and then split the result
			# into chunks of a fixed size, which we will call block_size. To do this, we
			# will use the map method again, with the option batched=True. When we use batched=True,
			# the function we pass to map() will be passed multiple inputs at once, allowing us
			# to group them into more or fewer examples than we had in the input.
			# This allows us to create our new fixed-length samples. The advantage of this
			# method is that we don't lose a whole lot of content from the dataset compared to the
			# case where we simply tokenize with a pre-defined max_length.

			def group_texts(A	:      List[Any]     ):
						# Concatenate all texts.
						a					=    {k: sum(examples[k],						[]     ) for k in examples.keys()}
						a					=    len(concatenated_examples[list(examples.keys()     )[0]]     )
						# We drop the small remainder, though you could add padding instead if the model supports it
						# In this, as in all things, we advise you to follow your heart 🫀
						a					=    (total_length // args.max_length) * args.max_length
						# Split by chunks of max_len.
						a					=    {
						    k: [t[i : i + args.max_length] for i in range(0,						A,						args.max_length     )]
						    for k, t in concatenated_examples.items()
						}
						return result

			a					=    dataset_tokenized.map(A,						batched=A,						batch_size=1000,						num_proc=4     )

			a					=    0
			a					=    0
			for shard in range(0,						len(A     ),						args.shard_size     ):
						a					=    grouped_dataset[shard : shard + args.shard_size]
						a					=    len(dataset_snapshot["input_ids"]     )
						a					=    os.path.join(A,						F"""dataset-{shard_count}-{records_containing}.tfrecord"""     )
						a					=    get_serialized_examples(A     )

						with tf.io.TFRecordWriter(A     ) as out_file:
									for i in range(len(A     )     ):
												a					=    serialized_examples[i]
												out_file.write(A     )
									print("Wrote file {} containing {} records".format(A,						A     )     )

						shard_count += 1
						total_records += records_containing

			with open(F"""split-{args.split}-records-count.txt""",						"w"     ) as f:
						print(F"""Total {args.split} records: {total_records}""",						file=A     )


if __name__ == "__main__":
		__lowerCAmelCase	:       Optional[int]     =	parse_args()
		main(args)






