





import gzip
import hashlib
import json
import multiprocessing
import os
import re
import shutil
import time
from pathlib import Path

import numpy as np
from arguments import PreprocessingArguments
from datasets import load_dataset
from minhash_deduplication import deduplicate_dataset

from transformers import AutoTokenizer, HfArgumentParser


lowerCAmelCase_						   =    re.compile(r"""\s+""")


def     lowerCamelCase_	(      lowerCAmelCase:	List[str]				)->		Dict:
  return {"hash": hashlib.mda(re.sub(lowerCAmelCase    ,							''    ,							example['content']				).encode('utf-8'				)				).hexdigest()}


def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]				)->		Optional[int]:
  _snake_case   :				Optional[int]														=	[len(lowerCAmelCase				) for line in example['content'].splitlines()]
  return {"line_mean": np.mean(lowerCAmelCase				), "line_max": max(lowerCAmelCase				)}


def     lowerCamelCase_	(      lowerCAmelCase:	List[str]				)->		int:
  _snake_case   :				str														=	np.mean([c.isalnum() for c in example['content']]				)
  return {"alpha_frac": alpha_frac}


def     lowerCamelCase_	(      lowerCAmelCase:	Tuple    ,							lowerCAmelCase:	Union[str, Any]				)->		int:
  if example["hash"] in uniques:
    uniques.remove(example['hash']				)
    return True
  else:
    return False


def     lowerCamelCase_	(      lowerCAmelCase:	str    ,							lowerCAmelCase:	str=5				)->		List[str]:
  _snake_case   :				Dict														=	['auto-generated', 'autogenerated', 'automatically generated']
  _snake_case   :				Dict														=	example['content'].splitlines()
  for _, line in zip(range(lowerCAmelCase				)    ,							lowerCAmelCase				):
    for keyword in keywords:
      if keyword in line.lower():
        return {"autogenerated": True}
  else:
    return {"autogenerated": False}


def     lowerCamelCase_	(      lowerCAmelCase:	int    ,							lowerCAmelCase:	Any=5    ,							lowerCAmelCase:	List[Any]=0.0_5				)->		List[str]:
  _snake_case   :				Tuple														=	['unit tests', 'test file', 'configuration file']
  _snake_case   :				Any														=	example['content'].splitlines()
  _snake_case   :				List[str]														=	0
  _snake_case   :				Tuple														=	0
  # first test
  for _, line in zip(range(lowerCAmelCase				)    ,							lowerCAmelCase				):
    for keyword in keywords:
      if keyword in line.lower():
        return {"config_or_test": True}
    # second test
  _snake_case   :				List[str]														=	example['content'].count('\n'				)
  _snake_case   :				Tuple														=	int(coeff * nlines				)
  for line in lines:
    count_config += line.lower().count('config'				)
    count_test += line.lower().count('test'				)
    if count_config > threshold or count_test > threshold:
      return {"config_or_test": True}
  return {"config_or_test": False}


def     lowerCamelCase_	(      lowerCAmelCase:	int				)->		Dict:
  _snake_case   :				str														=	['def ', 'class ', 'for ', 'while ']
  _snake_case   :				List[Any]														=	example['content'].splitlines()
  for line in lines:
    for keyword in keywords:
      if keyword in line.lower():
        return {"has_no_keywords": False}
  return {"has_no_keywords": True}


def     lowerCamelCase_	(      lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Dict=4				)->		int:
  _snake_case   :				Tuple														=	example['content'].splitlines()
  _snake_case   :				Optional[int]														=	0
  for line in lines:
    counter += line.lower().count('='				)
    if counter > minimum:
      return {"has_few_assignments": False}
  return {"has_few_assignments": True}


def     lowerCamelCase_	(      lowerCAmelCase:	int				)->		List[str]:
  _snake_case   :				int														=	tokenizer(example['content']    ,							truncation=lowerCAmelCase				)['input_ids']
  _snake_case   :				int														=	len(example['content']				) / len(lowerCAmelCase				)
  return {"ratio": ratio}


def     lowerCamelCase_	(      lowerCAmelCase:	Optional[Any]				)->		Dict:
  _snake_case   :				Optional[int]														=	{}
  results.update(get_hash(lowerCAmelCase				)				)
  results.update(line_stats(lowerCAmelCase				)				)
  results.update(alpha_stats(lowerCAmelCase				)				)
  results.update(char_token_ratio(lowerCAmelCase				)				)
  results.update(is_autogenerated(lowerCAmelCase				)				)
  results.update(is_config_or_test(lowerCAmelCase				)				)
  results.update(has_no_keywords(lowerCAmelCase				)				)
  results.update(has_few_assignments(lowerCAmelCase				)				)
  return results


def     lowerCamelCase_	(      lowerCAmelCase:	Dict    ,							lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	Optional[Any]				)->		Dict:
  if not check_uniques(lowerCAmelCase    ,							lowerCAmelCase				):
    return False
  elif example["autogenerated"]:
    return False
  elif example["line_max"] > args.line_max:
    return False
  elif example["line_mean"] > args.line_mean:
    return False
  elif example["alpha_frac"] < args.alpha_frac:
    return False
  elif example["ratio"] < args.min_token_ratio:
    return False
  elif example["config_or_test"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_no_keywords"] and np.random.rand() <= args.filter_proba:
    return False
  elif example["has_few_assignments"]:
    return False
  else:
    return True


def     lowerCamelCase_	(      lowerCAmelCase:	Optional[int]				)->		Any:
  with open(lowerCAmelCase    ,							'rb'				) as f_in:
    with gzip.open(str(lowerCAmelCase				) + '.gz'    ,							'wb'    ,							compresslevel=6				) as f_out:
      shutil.copyfileobj(lowerCAmelCase    ,							lowerCAmelCase				)
  os.unlink(lowerCAmelCase				)


# Settings
lowerCAmelCase_						   =    HfArgumentParser(PreprocessingArguments)
lowerCAmelCase_						   =    parser.parse_args()
if args.num_workers is None:
      lowerCAmelCase_						   =    multiprocessing.cpu_count()
lowerCAmelCase_						   =    AutoTokenizer.from_pretrained(args.tokenizer_dir)

# Load dataset
lowerCAmelCase_						   =    time.time()
lowerCAmelCase_						   =    load_dataset(args.dataset_name, split="""train""")
print(F"""Time to load dataset: {time.time()-t_start:.2f}""")

# Run preprocessing
lowerCAmelCase_						   =    time.time()
lowerCAmelCase_						   =    ds.map(preprocess, num_proc=args.num_workers)
print(F"""Time to preprocess dataset: {time.time()-t_start:.2f}""")

# Deduplicate hashes
lowerCAmelCase_						   =    set(ds.unique("""hash"""))
lowerCAmelCase_						   =    len(uniques) / len(ds)
print(F"""Fraction of duplicates: {1-frac:.2%}""")

# Deduplicate data and apply heuristics
lowerCAmelCase_						   =    time.time()
lowerCAmelCase_						   =    ds.filter(filter, fn_kwargs={"""uniques""": uniques, """args""": args})
print(F"""Time to filter dataset: {time.time()-t_start:.2f}""")
print(F"""Size of filtered dataset: {len(ds_filter)}""")

# Deduplicate with minhash and jaccard similarity
if args.near_deduplication:
      lowerCAmelCase_						   =    time.time()
      lowerCAmelCase_   ,							lowerCAmelCase_						   =    deduplicate_dataset(ds_filter, args.jaccard_threshold)
      print(F"""Time to deduplicate dataset: {time.time()-t_start:.2f}""")
      print(F"""Size of deduplicate dataset: {len(ds_filter)}""")

# Save data in batches of samples_per_file
lowerCAmelCase_						   =    Path(args.output_dir)
output_dir.mkdir(exist_ok=True)

# save duplicate_clusters in the output_dir as artifacts
# not sure it is the right place the save it
if args.near_deduplication:
      with open(output_dir / """duplicate_clusters.json""", """w""") as f:
            json.dump(duplicate_clusters, f)

lowerCAmelCase_						   =    output_dir / """data"""
data_dir.mkdir(exist_ok=True)

lowerCAmelCase_						   =    time.time()
for file_number, index in enumerate(range(0, len(ds_filter), args.samples_per_file)):
      lowerCAmelCase_						   =    str(data_dir / F"""file-{file_number+1:012}.json""")
      lowerCAmelCase_						   =    min(len(ds_filter), index + args.samples_per_file)
      ds_filter.select(list(range(index, end_index))).to_json(file_path)
      compress_file(file_path)







print(F"""Time to save dataset: {time.time()-t_start:.2f}""")











from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin



@dataclass
class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       torch.FloatTensor




class        _lowerCAmelCase    (			UpperCAmelCase_							,    UpperCAmelCase_							):


  '''simple docstring'''







  @register_to_config
  def __init__(      self      :					str      ,				UpperCamelCase      :					int = 32      ,				UpperCamelCase      :					int = 64      ,				UpperCamelCase      :					int = 20      ,				UpperCamelCase      :					int = 7_68      ,				UpperCamelCase      :					Optional[int]=77      ,				UpperCamelCase      :					int=4      ,				UpperCamelCase      :					float = 0.0      ,				UpperCamelCase      :					str = "silu"      ,				UpperCamelCase      :					Optional[str] = None      ,				UpperCamelCase      :					Optional[str] = None      ,				UpperCamelCase      :					Optional[str] = "linear"      ,				UpperCamelCase      :					Optional[str] = "prd"      ,				UpperCamelCase      :					Optional[int] = None      ,				UpperCamelCase      :					Optional[int] = None      ,				UpperCamelCase      :					Optional[int] = None      ,				):


    '''simple docstring'''





    super().__init__()
    _snake_case   :				str														=	num_attention_heads
    _snake_case   :				Optional[int]														=	attention_head_dim
    _snake_case   :				Any														=	num_attention_heads * attention_head_dim
    _snake_case   :				List[Any]														=	additional_embeddings

    _snake_case   :				List[str]														=	time_embed_dim or inner_dim
    _snake_case   :				int														=	embedding_proj_dim or embedding_dim
    _snake_case   :				List[Any]														=	clip_embed_dim or embedding_dim

    _snake_case   :				Optional[Any]														=	Timesteps(UpperCamelCase      ,				UpperCamelCase      ,				0							)
    _snake_case   :				List[Any]														=	TimestepEmbedding(UpperCamelCase      ,				UpperCamelCase      ,				out_dim=UpperCamelCase      ,				act_fn=UpperCamelCase							)

    _snake_case   :				Optional[int]														=	nn.Linear(UpperCamelCase      ,				UpperCamelCase							)

    if embedding_proj_norm_type is None:
      _snake_case   :				str														=	None
    elif embedding_proj_norm_type == "layer":
      _snake_case   :				List[Any]														=	nn.LayerNorm(UpperCamelCase							)
    else:
      raise ValueError(f"""unsupported embedding_proj_norm_type: {embedding_proj_norm_type}"""							)

    _snake_case   :				str														=	nn.Linear(UpperCamelCase      ,				UpperCamelCase							)

    if encoder_hid_proj_type is None:
      _snake_case   :				Any														=	None
    elif encoder_hid_proj_type == "linear":
      _snake_case   :				Optional[int]														=	nn.Linear(UpperCamelCase      ,				UpperCamelCase							)
    else:
      raise ValueError(f"""unsupported encoder_hid_proj_type: {encoder_hid_proj_type}"""							)

    _snake_case   :				List[str]														=	nn.Parameter(torch.zeros(1      ,				num_embeddings + additional_embeddings      ,				UpperCamelCase							)							)

    if added_emb_type == "prd":
      _snake_case   :				str														=	nn.Parameter(torch.zeros(1      ,				1      ,				UpperCamelCase							)							)
    elif added_emb_type is None:
      _snake_case   :				Dict														=	None
    else:
      raise ValueError(
          f"""`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `'prd'` or `None`."""							)

    _snake_case   :				Optional[int]														=	nn.ModuleList(
        [
            BasicTransformerBlock(
                UpperCamelCase      ,				UpperCamelCase      ,				UpperCamelCase      ,				dropout=UpperCamelCase      ,				activation_fn='gelu'      ,				attention_bias=UpperCamelCase      ,				)
            for d in range(UpperCamelCase							)
        ]							)

    if norm_in_type == "layer":
      _snake_case   :				Optional[int]														=	nn.LayerNorm(UpperCamelCase							)
    elif norm_in_type is None:
      _snake_case   :				Optional[Any]														=	None
    else:
      raise ValueError(f"""Unsupported norm_in_type: {norm_in_type}."""							)

    _snake_case   :				Optional[Any]														=	nn.LayerNorm(UpperCamelCase							)

    _snake_case   :				Union[str, Any]														=	nn.Linear(UpperCamelCase      ,				UpperCamelCase							)

    _snake_case   :				List[Any]														=	torch.full(
        [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings]      ,				-1_00_00.0							)
    causal_attention_mask.triu_(1							)
    _snake_case   :				Optional[Any]														=	causal_attention_mask[None, ...]
    self.register_buffer('causal_attention_mask'      ,				UpperCamelCase      ,				persistent=UpperCamelCase							)

    _snake_case   :				str														=	nn.Parameter(torch.zeros(1      ,				UpperCamelCase							)							)
    _snake_case   :				List[str]														=	nn.Parameter(torch.zeros(1      ,				UpperCamelCase							)							)







  @property
  # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	{}

    def fn_recursive_add_processors(UpperCamelCase      :					str      ,				UpperCamelCase      :					torch.nn.Module      ,				UpperCamelCase      :					Dict[str, AttentionProcessor]							):
      if hasattr(UpperCamelCase      ,				'set_processor'							):
        _snake_case   :				Tuple														=	module.processor

      for sub_name, child in module.named_children():
        fn_recursive_add_processors(f"""{name}.{sub_name}"""      ,				UpperCamelCase      ,				UpperCamelCase							)

      return processors

    for name, module in self.named_children():
      fn_recursive_add_processors(UpperCamelCase      ,				UpperCamelCase      ,				UpperCamelCase							)

    return processors







  def UpperCamelCase_ (      self      :					List[Any]      ,				UpperCamelCase      :					Union[AttentionProcessor, Dict[str, AttentionProcessor]]							):


    '''simple docstring'''





    _snake_case   :				Optional[int]														=	len(self.attn_processors.keys()							)

    if isinstance(UpperCamelCase      ,				UpperCamelCase							) and len(UpperCamelCase							) != count:
      raise ValueError(
          f"""A dict of processors was passed, but the number of processors {len(UpperCamelCase							)} does not match the"""
          f""" number of attention layers: {count}. Please make sure to pass {count} processor classes."""							)

    def fn_recursive_attn_processor(UpperCamelCase      :					str      ,				UpperCamelCase      :					torch.nn.Module      ,				UpperCamelCase      :					Union[str, Any]							):
      if hasattr(UpperCamelCase      ,				'set_processor'							):
        if not isinstance(UpperCamelCase      ,				UpperCamelCase							):
          module.set_processor(UpperCamelCase							)
        else:
          module.set_processor(processor.pop(f"""{name}.processor"""							)							)

      for sub_name, child in module.named_children():
        fn_recursive_attn_processor(f"""{name}.{sub_name}"""      ,				UpperCamelCase      ,				UpperCamelCase							)

    for name, module in self.named_children():
      fn_recursive_attn_processor(UpperCamelCase      ,				UpperCamelCase      ,				UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    self.set_attn_processor(AttnProcessor()							)







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					Any      ,				UpperCamelCase      :					Union[torch.Tensor, float, int]      ,				UpperCamelCase      :					torch.FloatTensor      ,				UpperCamelCase      :					Optional[torch.FloatTensor] = None      ,				UpperCamelCase      :					Optional[torch.BoolTensor] = None      ,				UpperCamelCase      :					bool = True      ,				):


    '''simple docstring'''





    _snake_case   :				Dict														=	hidden_states.shape[0]

    _snake_case   :				str														=	timestep
    if not torch.is_tensor(UpperCamelCase							):
      _snake_case   :				Dict														=	torch.tensor([timesteps]      ,				dtype=torch.long      ,				device=hidden_states.device							)
    elif torch.is_tensor(UpperCamelCase							) and len(timesteps.shape							) == 0:
      _snake_case   :				Tuple														=	timesteps[None].to(hidden_states.device							)

    # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
    _snake_case   :				Optional[int]														=	timesteps * torch.ones(UpperCamelCase      ,				dtype=timesteps.dtype      ,				device=timesteps.device							)

    _snake_case   :				Union[str, Any]														=	self.time_proj(UpperCamelCase							)

    # timesteps does not contain any weights and will always return f32 tensors
    # but time_embedding might be fp16, so we need to cast here.
    _snake_case   :				Tuple														=	timesteps_projected.to(dtype=self.dtype							)
    _snake_case   :				List[Any]														=	self.time_embedding(UpperCamelCase							)

    if self.embedding_proj_norm is not None:
      _snake_case   :				Optional[Any]														=	self.embedding_proj_norm(UpperCamelCase							)

    _snake_case   :				Union[str, Any]														=	self.embedding_proj(UpperCamelCase							)
    if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
      _snake_case   :				Dict														=	self.encoder_hidden_states_proj(UpperCamelCase							)
    elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
      raise ValueError('`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set'							)

    _snake_case   :				str														=	self.proj_in(UpperCamelCase							)

    _snake_case   :				int														=	self.positional_embedding.to(hidden_states.dtype							)

    _snake_case   :				Optional[int]														=	[]
    _snake_case   :				List[Any]														=	0

    if encoder_hidden_states is not None:
      additional_embeds.append(UpperCamelCase							)
      additional_embeddings_len += encoder_hidden_states.shape[1]

    if len(proj_embeddings.shape							) == 2:
      _snake_case   :				str														=	proj_embeddings[:, None, :]

    if len(hidden_states.shape							) == 2:
      _snake_case   :				str														=	hidden_states[:, None, :]

    _snake_case   :				str														=	additional_embeds + [
        proj_embeddings,
        time_embeddings[:, None, :],
        hidden_states,
    ]

    if self.prd_embedding is not None:
      _snake_case   :				int														=	self.prd_embedding.to(hidden_states.dtype							).expand(UpperCamelCase      ,				-1      ,				-1							)
      additional_embeds.append(UpperCamelCase							)

    _snake_case   :				Optional[int]														=	torch.cat(
        UpperCamelCase      ,				dim=1      ,				)

    # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
    _snake_case   :				Tuple														=	additional_embeddings_len + proj_embeddings.shape[1] + 1
    if positional_embeddings.shape[1] < hidden_states.shape[1]:
      _snake_case   :				Optional[Any]														=	F.pad(
          UpperCamelCase      ,				(
              0,
              0,
              additional_embeddings_len,
              self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
          )      ,				value=0.0      ,				)

    _snake_case   :				Optional[Any]														=	hidden_states + positional_embeddings

    if attention_mask is not None:
      _snake_case   :				Any														=	(1 - attention_mask.to(hidden_states.dtype							)) * -1_00_00.0
      _snake_case   :				Tuple														=	F.pad(UpperCamelCase      ,				(0, self.additional_embeddings)      ,				value=0.0							)
      _snake_case   :				int														=	(attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype							)
      _snake_case   :				str														=	attention_mask.repeat_interleave(self.config.num_attention_heads      ,				dim=0							)

    if self.norm_in is not None:
      _snake_case   :				Tuple														=	self.norm_in(UpperCamelCase							)

    for block in self.transformer_blocks:
      _snake_case   :				Any														=	block(UpperCamelCase      ,				attention_mask=UpperCamelCase							)

    _snake_case   :				Dict														=	self.norm_out(UpperCamelCase							)

    if self.prd_embedding is not None:
      _snake_case   :				str														=	hidden_states[:, -1]
    else:
      _snake_case   :				Any														=	hidden_states[:, additional_embeddings_len:]

    _snake_case   :				List[Any]														=	self.proj_to_clip_embeddings(UpperCamelCase							)

    if not return_dict:
      return (predicted_image_embedding,)

    return PriorTransformerOutput(predicted_image_embedding=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				List[Any]														=	(prior_latents * self.clip_std) + self.clip_mean
    return prior_latents




