
'''simple docstring'''







import json
import multiprocessing as mp
import re
from collections import defaultdict
from functools import partial
from typing import Dict, List, Optional, Set, Tuple, Type

from datasets import Dataset
from datasketch import MinHash, MinHashLSH
from dpu_utils.utils.iterators import ThreadedIterator
from tqdm import tqdm


__lowerCamelCase		       =    re.compile('''[^A-Za-z_0-9]''')
# parameters used in DuplicationIndex
__lowerCamelCase		       =    10
__lowerCamelCase		       =    256
def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			Dict:





       if len(SCREAMING_SNAKE_CASE__				) < MIN_NUM_TOKENS:
              return None
       A_						=      MinHash(num_perm=SCREAMING_SNAKE_CASE__				)
       for token in set(SCREAMING_SNAKE_CASE__				):
              min_hash.update(token.encode()				)
       return min_hash
def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			Union[str, Any]:
       return {t for t in NON_ALPHA.split(SCREAMING_SNAKE_CASE__				) if len(t.strip()				) > 0}


class 					A__      :



    def __init__(					self		,				*,
        UpperCamelCase__ = 0.85		,				)						->      Any:





           '''simple docstring'''
           A_						=      duplication_jaccard_threshold
           A_						=      NUM_PERM
           A_						=      MinHashLSH(threshold=self._duplication_jaccard_threshold		,				num_perm=self._num_perm       )

           A_						=      defaultdict(lowercase__       )



    def 					snake_case_			(					self		,				UpperCamelCase__		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      self._index.query(lowercase__       )
           if code_key in self._index.keys:
                  print(f'''Duplicate key {code_key}'''       )
                  return

           self._index.insert(lowercase__		,				lowercase__       )
           if len(lowercase__       ) > 0:
                  for base_duplicate in close_duplicates:
                         if base_duplicate in self._duplicate_clusters:
                                self._duplicate_clusters[base_duplicate].add(lowercase__       )
                                break
                  else:
                         self._duplicate_clusters[close_duplicates[0]].add(lowercase__       )



    def 					snake_case_			(					self       )						->      int:





           '''simple docstring'''
           A_						=      []
           for base, duplicates in self._duplicate_clusters.items():
                  A_						=      [base] + list(lowercase__       )
                  # reformat the cluster to be a list of dict
                  A_						=      [{"""base_index""": el[0], """repo_name""": el[1], """path""": el[2]} for el in cluster]
                  duplicate_clusters.append(lowercase__       )
           return duplicate_clusters







    def 					snake_case_			(					self		,				UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      self.get_duplicate_clusters()
           with open(lowercase__		,				"""w"""       ) as f:
                  json.dump(lowercase__		,				lowercase__       )

def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			str:
       A_						=      element
       A_						=      get_min_hash([t for t in NON_ALPHA.split(data["""content"""]				) if len(t.strip()				) > 0]				)
       if min_hash is not None:
              return (index, data["repo_name"], data["path"]), min_hash
def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			Optional[int]:





       with mp.Pool() as pool:
              for data in pool.imap_unordered(
                  _compute_min_hash,       ThreadedIterator(SCREAMING_SNAKE_CASE__,       max_queue_size=1_00_00				),       chunksize=1_00,       ):
                     if data is not None:
                            yield data
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			Dict:
       A_						=      DuplicationIndex(duplication_jaccard_threshold=SCREAMING_SNAKE_CASE__				)

       for filename, min_hash in tqdm(ThreadedIterator(minhash_iter(enumerate(SCREAMING_SNAKE_CASE__				)				),       max_queue_size=1_00				)				):
              di.add(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__				)

       # Returns a List[Cluster] where Cluster is List[str] with the filenames.
       return di.get_duplicate_clusters()
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			List[Any]:
       A_						=      get_tokens(SCREAMING_SNAKE_CASE__				)
       A_						=      get_tokens(SCREAMING_SNAKE_CASE__				)
       return len(tokensa & tokensa				) / len(tokensa | tokensa				)


__lowerCamelCase		       =    None
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__				)			->			str:
       A_						=      []
       for elementa in cluster:
              A_						=      _shared_dataset[elementa["""base_index"""]]["""content"""]
              for elementa in extremes:
                     A_						=      _shared_dataset[elementa["""base_index"""]]["""content"""]
                     if jaccard_similarity(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__				) >= jaccard_threshold:
                            elementa["copies"] += 1
                            break
              else:
                     A_						=      1
                     extremes.append(SCREAMING_SNAKE_CASE__				)
       return extremes
def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__,       UpperCAmelCase__				)			->			str:
       global _shared_dataset
       A_						=      dataset
       A_						=      []
       A_						=      partial(_find_cluster_extremes_shared,       jaccard_threshold=SCREAMING_SNAKE_CASE__				)
       with mp.Pool() as pool:
              for extremes in tqdm(
                  pool.imap_unordered(
                      SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__,       ),       total=len(SCREAMING_SNAKE_CASE__				),       ):
                     extremes_list.append(SCREAMING_SNAKE_CASE__				)
       return extremes_list







def       UpperCAmelCase__    (      UpperCAmelCase__,       UpperCAmelCase__ = 0.85				)			->			List[str]:
       A_						=      make_duplicate_clusters(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__				)
       A_						=      {x["""base_index"""] for cluster in duplicate_clusters for x in cluster}
       A_						=      {}
       A_						=      find_extremes(SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__,       SCREAMING_SNAKE_CASE__				)
       for extremes in extremes_clusters:
              for element in extremes:
                     A_						=      element
       A_						=      duplicate_indices - set(extreme_dict.keys()				)
       A_						=      dataset.filter(lambda UpperCAmelCase__,       UpperCAmelCase__				: idx not in remove_indices,       with_indices=SCREAMING_SNAKE_CASE__				)

       # update duplicate_clusters
       for cluster in duplicate_clusters:
              for element in cluster:
                     A_						=      element["""base_index"""] in extreme_dict
                     if element["is_extreme"]:
                            A_						=      extreme_dict[element["""base_index"""]]["""copies"""]

       print(F'''Original dataset size: {len(SCREAMING_SNAKE_CASE__				)}'''				)
       print(F'''Number of duplicate clusters: {len(SCREAMING_SNAKE_CASE__				)}'''				)
       print(F'''Files in duplicate cluster: {len(SCREAMING_SNAKE_CASE__				)}'''				)
       print(F'''Unique files in duplicate cluster: {len(SCREAMING_SNAKE_CASE__				)}'''				)
       print(F'''Filtered dataset size: {len(SCREAMING_SNAKE_CASE__				)}'''				)

       return ds_filter, duplicate_clusters



'''simple docstring'''







import importlib.util
import os
import platform
from argparse import ArgumentParser

import huggingface_hub

from .. import __version__ as version
from ..utils import (
    is_accelerate_available,
    is_flax_available,
    is_safetensors_available,
    is_tf_available,
    is_torch_available,
)
from . import BaseTransformersCLICommand
def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			Tuple:
       return EnvironmentCommand()
def       UpperCAmelCase__    (      UpperCAmelCase__				)			->			int:
       return EnvironmentCommand(args.accelerate_config_file				)







class 					A__      (      _snake_case			):



    @staticmethod
    def 					snake_case_			(					UpperCamelCase__       )						->      Union[str, Any]:





           '''simple docstring'''
           A_						=      parser.add_parser("""env"""       )
           download_parser.set_defaults(func=UpperCamelCase__       )
           download_parser.add_argument(
               """--accelerate-config_file"""		,				default=UpperCamelCase__		,				help="""The accelerate config file to use for the default values in the launching script."""		,				)
           download_parser.set_defaults(func=UpperCamelCase__       )



    def __init__(					self		,				UpperCamelCase__		,				*UpperCamelCase__       )						->      None:





           '''simple docstring'''
           A_						=      accelerate_config_file



    def 					snake_case_			(					self       )						->      List[str]:





           '''simple docstring'''
           A_						=      """not installed"""
           if is_safetensors_available():
                  import safetensors

                  A_						=      safetensors.__version__
           elif importlib.util.find_spec("""safetensors"""       ) is not None:
                  import safetensors

                  A_						=      f'''{safetensors.__version__} but is ignored because of PyTorch version too old.'''

           A_						=      """not installed"""
           A_						=      A_						=      """not found"""
           if is_accelerate_available():
                  import accelerate
                  from accelerate.commands.config import default_config_file, load_config_from_file

                  A_						=      accelerate.__version__
                  # Get the default from the config file.
                  if self._accelerate_config_file is not None or os.path.isfile(UpperCamelCase__       ):
                         A_						=      load_config_from_file(self._accelerate_config_file       ).to_dict()

                  A_						=      (
                      """\n""".join([f'''\t- {prop}: {val}''' for prop, val in accelerate_config.items()]       )
                      if isinstance(UpperCamelCase__		,				UpperCamelCase__       )
                      else f'''\t{accelerate_config}'''
                  )

           A_						=      """not installed"""
           A_						=      """NA"""
           if is_torch_available():
                  import torch

                  A_						=      torch.__version__
                  A_						=      torch.cuda.is_available()

           A_						=      """not installed"""
           A_						=      """NA"""
           if is_tf_available():
                  import tensorflow as tf

                  A_						=      tf.__version__
                  try:
                         # deprecated in v2.1
                         A_						=      tf.test.is_gpu_available()
                  except AttributeError:
                         # returns list of devices, convert to bool
                         A_						=      bool(tf.config.list_physical_devices("""GPU"""       )       )

           A_						=      """not installed"""
           A_						=      """not installed"""
           A_						=      """not installed"""
           A_						=      """NA"""
           if is_flax_available():
                  import flax
                  import jax
                  import jaxlib

                  A_						=      flax.__version__
                  A_						=      jax.__version__
                  A_						=      jaxlib.__version__
                  A_						=      jax.lib.xla_bridge.get_backend().platform

           A_						=      {
               """`transformers` version""": version,
               """Platform""": platform.platform(),
               """Python version""": platform.python_version(),
               """Huggingface_hub version""": huggingface_hub.__version__,
               """Safetensors version""": f'''{safetensors_version}''',
               """Accelerate version""": f'''{accelerate_version}''',
               """Accelerate config""": f'''{accelerate_config_str}''',
               """PyTorch version (GPU?)""": f'''{pt_version} ({pt_cuda_available})''',
               """Tensorflow version (GPU?)""": f'''{tf_version} ({tf_cuda_available})''',
               """Flax version (CPU?/GPU?/TPU?)""": f'''{flax_version} ({jax_backend})''',
               """Jax version""": f'''{jax_version}''',
               """JaxLib version""": f'''{jaxlib_version}''',
               """Using GPU in script?""": """<fill in>""",
               """Using distributed or parallel set-up in script?""": """<fill in>""",
           }

           print("""\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n"""       )
           print(self.format_dict(UpperCamelCase__       )       )

           return info







    @staticmethod
    def 					snake_case_			(					UpperCamelCase__       )						->      List[str]:





           '''simple docstring'''
           return "\n".join([f'''- {prop}: {val}''' for prop, val in d.items()]       ) + "\n"
