


"""simple docstring"""


import argparse
import json
from collections import OrderedDict
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    ConditionalDetrConfig,
    ConditionalDetrForObjectDetection,
    ConditionalDetrForSegmentation,
    ConditionalDetrImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
_snake_case          =							logging.get_logger(__name__)

# here we list all keys to be renamed (original name on the left, our name on the right)
_snake_case          =							[]
for i in range(6):
   # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
   rename_keys.append(
       (f"""transformer.encoder.layers.{i}.self_attn.out_proj.weight""", f"""encoder.layers.{i}.self_attn.out_proj.weight""")
   )
   rename_keys.append(
       (f"""transformer.encoder.layers.{i}.self_attn.out_proj.bias""", f"""encoder.layers.{i}.self_attn.out_proj.bias""")
   )
   rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.weight""", f"""encoder.layers.{i}.fc1.weight"""))
   rename_keys.append((f"""transformer.encoder.layers.{i}.linear1.bias""", f"""encoder.layers.{i}.fc1.bias"""))
   rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.weight""", f"""encoder.layers.{i}.fc2.weight"""))
   rename_keys.append((f"""transformer.encoder.layers.{i}.linear2.bias""", f"""encoder.layers.{i}.fc2.bias"""))
   rename_keys.append(
       (f"""transformer.encoder.layers.{i}.norm1.weight""", f"""encoder.layers.{i}.self_attn_layer_norm.weight""")
   )
   rename_keys.append((f"""transformer.encoder.layers.{i}.norm1.bias""", f"""encoder.layers.{i}.self_attn_layer_norm.bias"""))
   rename_keys.append((f"""transformer.encoder.layers.{i}.norm2.weight""", f"""encoder.layers.{i}.final_layer_norm.weight"""))
   rename_keys.append((f"""transformer.encoder.layers.{i}.norm2.bias""", f"""encoder.layers.{i}.final_layer_norm.bias"""))
   # decoder layers: 2 times output projection, 2 feedforward neural networks and 3 layernorms
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.self_attn.out_proj.weight""", f"""decoder.layers.{i}.self_attn.out_proj.weight""")
   )
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.self_attn.out_proj.bias""", f"""decoder.layers.{i}.self_attn.out_proj.bias""")
   )
   rename_keys.append(
       (
           f"""transformer.decoder.layers.{i}.cross_attn.out_proj.weight""",
           f"""decoder.layers.{i}.encoder_attn.out_proj.weight""",
       )
   )
   rename_keys.append(
       (
           f"""transformer.decoder.layers.{i}.cross_attn.out_proj.bias""",
           f"""decoder.layers.{i}.encoder_attn.out_proj.bias""",
       )
   )
   rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.weight""", f"""decoder.layers.{i}.fc1.weight"""))
   rename_keys.append((f"""transformer.decoder.layers.{i}.linear1.bias""", f"""decoder.layers.{i}.fc1.bias"""))
   rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.weight""", f"""decoder.layers.{i}.fc2.weight"""))
   rename_keys.append((f"""transformer.decoder.layers.{i}.linear2.bias""", f"""decoder.layers.{i}.fc2.bias"""))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.norm1.weight""", f"""decoder.layers.{i}.self_attn_layer_norm.weight""")
   )
   rename_keys.append((f"""transformer.decoder.layers.{i}.norm1.bias""", f"""decoder.layers.{i}.self_attn_layer_norm.bias"""))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.norm2.weight""", f"""decoder.layers.{i}.encoder_attn_layer_norm.weight""")
   )
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.norm2.bias""", f"""decoder.layers.{i}.encoder_attn_layer_norm.bias""")
   )
   rename_keys.append((f"""transformer.decoder.layers.{i}.norm3.weight""", f"""decoder.layers.{i}.final_layer_norm.weight"""))
   rename_keys.append((f"""transformer.decoder.layers.{i}.norm3.bias""", f"""decoder.layers.{i}.final_layer_norm.bias"""))

   # q, k, v projections in self/cross-attention in decoder for conditional DETR
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.sa_qcontent_proj.weight""", f"""decoder.layers.{i}.sa_qcontent_proj.weight""")
   )
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.sa_kcontent_proj.weight""", f"""decoder.layers.{i}.sa_kcontent_proj.weight""")
   )
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.sa_qpos_proj.weight""", f"""decoder.layers.{i}.sa_qpos_proj.weight""")
   )
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.sa_kpos_proj.weight""", f"""decoder.layers.{i}.sa_kpos_proj.weight""")
   )
   rename_keys.append((f"""transformer.decoder.layers.{i}.sa_v_proj.weight""", f"""decoder.layers.{i}.sa_v_proj.weight"""))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.ca_qcontent_proj.weight""", f"""decoder.layers.{i}.ca_qcontent_proj.weight""")
   )
   # rename_keys.append((f"transformer.decoder.layers.{i}.ca_qpos_proj.weight", f"decoder.layers.{i}.ca_qpos_proj.weight"))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.ca_kcontent_proj.weight""", f"""decoder.layers.{i}.ca_kcontent_proj.weight""")
   )
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.ca_kpos_proj.weight""", f"""decoder.layers.{i}.ca_kpos_proj.weight""")
   )
   rename_keys.append((f"""transformer.decoder.layers.{i}.ca_v_proj.weight""", f"""decoder.layers.{i}.ca_v_proj.weight"""))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.ca_qpos_sine_proj.weight""", f"""decoder.layers.{i}.ca_qpos_sine_proj.weight""")
   )

   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.sa_qcontent_proj.bias""", f"""decoder.layers.{i}.sa_qcontent_proj.bias""")
   )
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.sa_kcontent_proj.bias""", f"""decoder.layers.{i}.sa_kcontent_proj.bias""")
   )
   rename_keys.append((f"""transformer.decoder.layers.{i}.sa_qpos_proj.bias""", f"""decoder.layers.{i}.sa_qpos_proj.bias"""))
   rename_keys.append((f"""transformer.decoder.layers.{i}.sa_kpos_proj.bias""", f"""decoder.layers.{i}.sa_kpos_proj.bias"""))
   rename_keys.append((f"""transformer.decoder.layers.{i}.sa_v_proj.bias""", f"""decoder.layers.{i}.sa_v_proj.bias"""))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.ca_qcontent_proj.bias""", f"""decoder.layers.{i}.ca_qcontent_proj.bias""")
   )
   # rename_keys.append((f"transformer.decoder.layers.{i}.ca_qpos_proj.bias", f"decoder.layers.{i}.ca_qpos_proj.bias"))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.ca_kcontent_proj.bias""", f"""decoder.layers.{i}.ca_kcontent_proj.bias""")
   )
   rename_keys.append((f"""transformer.decoder.layers.{i}.ca_kpos_proj.bias""", f"""decoder.layers.{i}.ca_kpos_proj.bias"""))
   rename_keys.append((f"""transformer.decoder.layers.{i}.ca_v_proj.bias""", f"""decoder.layers.{i}.ca_v_proj.bias"""))
   rename_keys.append(
       (f"""transformer.decoder.layers.{i}.ca_qpos_sine_proj.bias""", f"""decoder.layers.{i}.ca_qpos_sine_proj.bias""")
   )

# convolutional projection + query embeddings + layernorm of decoder + class and bounding box heads
# for conditional DETR, also convert reference point head and query scale MLP
rename_keys.extend(
    [
        ("input_proj.weight", "input_projection.weight"),
        ("input_proj.bias", "input_projection.bias"),
        ("query_embed.weight", "query_position_embeddings.weight"),
        ("transformer.decoder.norm.weight", "decoder.layernorm.weight"),
        ("transformer.decoder.norm.bias", "decoder.layernorm.bias"),
        ("class_embed.weight", "class_labels_classifier.weight"),
        ("class_embed.bias", "class_labels_classifier.bias"),
        ("bbox_embed.layers.0.weight", "bbox_predictor.layers.0.weight"),
        ("bbox_embed.layers.0.bias", "bbox_predictor.layers.0.bias"),
        ("bbox_embed.layers.1.weight", "bbox_predictor.layers.1.weight"),
        ("bbox_embed.layers.1.bias", "bbox_predictor.layers.1.bias"),
        ("bbox_embed.layers.2.weight", "bbox_predictor.layers.2.weight"),
        ("bbox_embed.layers.2.bias", "bbox_predictor.layers.2.bias"),
        ("transformer.decoder.ref_point_head.layers.0.weight", "decoder.ref_point_head.layers.0.weight"),
        ("transformer.decoder.ref_point_head.layers.0.bias", "decoder.ref_point_head.layers.0.bias"),
        ("transformer.decoder.ref_point_head.layers.1.weight", "decoder.ref_point_head.layers.1.weight"),
        ("transformer.decoder.ref_point_head.layers.1.bias", "decoder.ref_point_head.layers.1.bias"),
        ("transformer.decoder.query_scale.layers.0.weight", "decoder.query_scale.layers.0.weight"),
        ("transformer.decoder.query_scale.layers.0.bias", "decoder.query_scale.layers.0.bias"),
        ("transformer.decoder.query_scale.layers.1.weight", "decoder.query_scale.layers.1.weight"),
        ("transformer.decoder.query_scale.layers.1.bias", "decoder.query_scale.layers.1.bias"),
        ("transformer.decoder.layers.0.ca_qpos_proj.weight", "decoder.layers.0.ca_qpos_proj.weight"),
        ("transformer.decoder.layers.0.ca_qpos_proj.bias", "decoder.layers.0.ca_qpos_proj.bias"),
    ]
)



def   snake_case						(				_a:     Tuple		,   _a:     List[Any]		,   _a:     Union[str, Any]							)->		Optional[int]:
 '''simple docstring'''


 lowerCamelCase__           =		state_dict.pop(__snake_case							)
 lowerCamelCase__           =		val



def   snake_case						(				_a:     Optional[int]							)->		List[Any]:
 '''simple docstring'''


 lowerCamelCase__           =		OrderedDict()
 for key, value in state_dict.items():
  if "backbone.0.body" in key:
   lowerCamelCase__           =		key.replace('backbone.0.body'		,   'backbone.conv_encoder.model'							)
   lowerCamelCase__           =		value
  else:
   lowerCamelCase__           =		value

 return new_state_dict



def   snake_case						(				_a:     Optional[Any]		,   _a:     Optional[int]=False							)->		str:
 '''simple docstring'''


 lowerCamelCase__           =		""
 if is_panoptic:
  lowerCamelCase__           =		"conditional_detr."

 # first: transformer encoder
 for i in range(6							):
  # read in weights + bias of input projection layer (in PyTorch's MultiHeadAttention, this is a single matrix + bias)
  lowerCamelCase__           =		state_dict.pop(F'{prefix}transformer.encoder.layers.{i}.self_attn.in_proj_weight'							)
  lowerCamelCase__           =		state_dict.pop(F'{prefix}transformer.encoder.layers.{i}.self_attn.in_proj_bias'							)
  # next, add query, keys and values (in that order) to the state dict
  lowerCamelCase__           =		in_proj_weight[:256, :]
  lowerCamelCase__           =		in_proj_bias[:256]
  lowerCamelCase__           =		in_proj_weight[256:512, :]
  lowerCamelCase__           =		in_proj_bias[256:512]
  lowerCamelCase__           =		in_proj_weight[-256:, :]
  lowerCamelCase__           =		in_proj_bias[-256:]



def   snake_case						(				)->		Optional[int]:
 '''simple docstring'''


 lowerCamelCase__           =		"http://images.cocodataset.org/val2017/000000039769.jpg"
 lowerCamelCase__           =		Image.open(requests.get(__snake_case		,   stream=__snake_case							).raw							)

 return im



@torch.no_grad()
def   snake_case						(				_a:     Any		,   _a:     Optional[int]							)->		Dict:
 '''simple docstring'''


 lowerCamelCase__           =		ConditionalDetrConfig()
 # set backbone and dilation attributes
 if "resnet101" in model_name:
  lowerCamelCase__           =		"resnet101"
 if "dc5" in model_name:
  lowerCamelCase__           =		True
 lowerCamelCase__           =		"panoptic" in model_name
 if is_panoptic:
  lowerCamelCase__           =		250
 else:
  lowerCamelCase__           =		91
  lowerCamelCase__           =		"huggingface/label-files"
  lowerCamelCase__           =		"coco-detection-id2label.json"
  lowerCamelCase__           =		json.load(open(hf_hub_download(__snake_case		,   __snake_case		,   repo_type='dataset'							)		,   'r'							)							)
  lowerCamelCase__           =		{int(__snake_case							): v for k, v in idalabel.items()}
  lowerCamelCase__           =		idalabel
  lowerCamelCase__           =		{v: k for k, v in idalabel.items()}

 # load image processor
 lowerCamelCase__           =		"coco_panoptic" if is_panoptic else "coco_detection"
 lowerCamelCase__           =		ConditionalDetrImageProcessor(format=__snake_case							)

 # prepare image
 lowerCamelCase__           =		prepare_img()
 lowerCamelCase__           =		image_processor(images=__snake_case		,   return_tensors='pt'							)
 lowerCamelCase__           =		encoding["pixel_values"]

 logger.info(F'Converting model {model_name}...'							)

 # load original model from torch hub
 lowerCamelCase__           =		torch.hub.load('DeppMeng/ConditionalDETR'		,   __snake_case		,   pretrained=__snake_case							).eval()
 lowerCamelCase__           =		conditional_detr.state_dict()
 # rename keys
 for src, dest in rename_keys:
  if is_panoptic:
   lowerCamelCase__           =		"conditional_detr." + src
  rename_key(__snake_case		,   __snake_case		,   __snake_case							)
 lowerCamelCase__           =		rename_backbone_keys(__snake_case							)
 # query, key and value matrices need special treatment
 read_in_q_k_v(__snake_case		,   is_panoptic=__snake_case							)
 # important: we need to prepend a prefix to each of the base model keys as the head models use different attributes for them
 lowerCamelCase__           =		"conditional_detr.model." if is_panoptic else "model."
 for key in state_dict.copy().keys():
  if is_panoptic:
   if (
       key.startswith('conditional_detr'							)
       and not key.startswith('class_labels_classifier'							)
       and not key.startswith('bbox_predictor'							)
   ):
    lowerCamelCase__           =		state_dict.pop(__snake_case							)
    lowerCamelCase__           =		val
   elif "class_labels_classifier" in key or "bbox_predictor" in key:
    lowerCamelCase__           =		state_dict.pop(__snake_case							)
    lowerCamelCase__           =		val
   elif key.startswith('bbox_attention'							) or key.startswith('mask_head'							):
    continue
   else:
    lowerCamelCase__           =		state_dict.pop(__snake_case							)
    lowerCamelCase__           =		val
  else:
   if not key.startswith('class_labels_classifier'							) and not key.startswith('bbox_predictor'							):
    lowerCamelCase__           =		state_dict.pop(__snake_case							)
    lowerCamelCase__           =		val
    # finally, create HuggingFace model and load state dict
 lowerCamelCase__           =		ConditionalDetrForSegmentation(__snake_case							) if is_panoptic else ConditionalDetrForObjectDetection(__snake_case							)
 model.load_state_dict(__snake_case							)
 model.eval()
 model.push_to_hub(repo_id=__snake_case		,   organization='DepuMeng'		,   commit_message='Add model'							)
 # verify our conversion
 lowerCamelCase__           =		conditional_detr(__snake_case							)
 lowerCamelCase__           =		model(__snake_case							)
 assert torch.allclose(outputs.logits		,   original_outputs['pred_logits']		,   atol=1E-4							)
 assert torch.allclose(outputs.pred_boxes		,   original_outputs['pred_boxes']		,   atol=1E-4							)
 if is_panoptic:
  assert torch.allclose(outputs.pred_masks		,   original_outputs['pred_masks']		,   atol=1E-4							)

 # Save model and image processor
 logger.info(F'Saving PyTorch model and image processor to {pytorch_dump_folder_path}...'							)
 Path(__snake_case							).mkdir(exist_ok=__snake_case							)
 model.save_pretrained(__snake_case							)
 image_processor.save_pretrained(__snake_case							)


if __name__ == "__main__":
   _snake_case          =							argparse.ArgumentParser()

   parser.add_argument(
       "--model_name",
       default="conditional_detr_resnet50",
       type=str,
       help="Name of the CONDITIONAL_DETR model you'd like to convert.",
   )
   parser.add_argument(
       "--pytorch_dump_folder_path", default=None, type=str, help="Path to the folder to output PyTorch model."
   )
   _snake_case          =							parser.parse_args()
   convert_conditional_detr_checkpoint(args.model_name, args.pytorch_dump_folder_path)







"""simple docstring"""


import json
import multiprocessing as mp
import re
from collections import defaultdict
from functools import partial
from typing import Dict, List, Optional, Set, Tuple, Type

from datasets import Dataset
from datasketch import MinHash, MinHashLSH
from dpu_utils.utils.iterators import ThreadedIterator
from tqdm import tqdm


_snake_case          =							re.compile("[^A-Za-z_0-9]")
# parameters used in DuplicationIndex
_snake_case          =							10
_snake_case          =							256



def   snake_case						(				_a:     List[str]							)->		Optional[MinHash]:
      '''simple docstring'''


      if len(_a							) < MIN_NUM_TOKENS:
            return None
      lowerCamelCase__           =		MinHash(num_perm=_a							)
      for token in set(_a							):
            min_hash.update(token.encode()							)
      return min_hash



def   snake_case						(				_a:     str							)->		Set[str]:
      '''simple docstring'''


      return {t for t in NON_ALPHA.split(_a							) if len(t.strip()							) > 0}



class   _a      :
    def __init__( self							:					List[Any]							,					*,
        SCREAMING_SNAKE_CASE__							:					float = 0.85							,					):
          lowerCamelCase__           =		duplication_jaccard_threshold
          lowerCamelCase__           =		NUM_PERM
          lowerCamelCase__           =		MinHashLSH(threshold=self._duplication_jaccard_threshold							,					num_perm=self._num_perm				)

          lowerCamelCase__           =		defaultdict(SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					List[Any]							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					MinHash				):
          lowerCamelCase__           =		self._index.query(SCREAMING_SNAKE_CASE__				)
          if code_key in self._index.keys:
                print(F'Duplicate key {code_key}'				)
                return

          self._index.insert(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
          if len(SCREAMING_SNAKE_CASE__				) > 0:
                for base_duplicate in close_duplicates:
                      if base_duplicate in self._duplicate_clusters:
                            self._duplicate_clusters[base_duplicate].add(SCREAMING_SNAKE_CASE__				)
                            break
                else:
                      self._duplicate_clusters[close_duplicates[0]].add(SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					str				):
          lowerCamelCase__           =		[]
          for base, duplicates in self._duplicate_clusters.items():
                lowerCamelCase__           =		[base] + list(SCREAMING_SNAKE_CASE__				)
                # reformat the cluster to be a list of dict
                lowerCamelCase__           =		[{'base_index': el[0], 'repo_name': el[1], 'path': el[2]} for el in cluster]
                duplicate_clusters.append(SCREAMING_SNAKE_CASE__				)
          return duplicate_clusters


    def      _UpperCamelCase       ( self							:					List[str]							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]				):
          lowerCamelCase__           =		self.get_duplicate_clusters()
          with open(SCREAMING_SNAKE_CASE__							,					'w'				) as f:
                json.dump(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)





def   snake_case						(				_a:     Union[str, Any]							)->		Optional[int]:
      '''simple docstring'''


      lowerCamelCase__    ,  lowerCamelCase__           =		element
      lowerCamelCase__           =		get_min_hash([t for t in NON_ALPHA.split(data['content']							) if len(t.strip()							) > 0]							)
      if min_hash is not None:
            return (index, data["repo_name"], data["path"]), min_hash



def   snake_case						(				_a:     Type[Dataset]							)->		Tuple:
      '''simple docstring'''


      with mp.Pool() as pool:
            for data in pool.imap_unordered(
                _compute_min_hash		,   ThreadedIterator(_a		,   max_queue_size=10000							)		,   chunksize=100		,   ):
                  if data is not None:
                        yield data



def   snake_case						(				_a:     Type[Dataset]		,   _a:     float							)->		Optional[int]:
      '''simple docstring'''


      lowerCamelCase__           =		DuplicationIndex(duplication_jaccard_threshold=_a							)

      for filename, min_hash in tqdm(ThreadedIterator(minhash_iter(enumerate(_a							)							)		,   max_queue_size=100							)							):
            di.add(_a		,   _a							)

      # Returns a List[Cluster] where Cluster is List[str] with the filenames.
      return di.get_duplicate_clusters()



def   snake_case						(				_a:     str		,   _a:     str							)->		float:
      '''simple docstring'''


      lowerCamelCase__           =		get_tokens(_a							)
      lowerCamelCase__           =		get_tokens(_a							)
      return len(tokensa & tokensa							) / len(tokensa | tokensa							)


_snake_case          =							None



def   snake_case						(				_a:     Dict		,   _a:     Union[str, Any]							)->		List[str]:
      '''simple docstring'''


      lowerCamelCase__           =		[]
      for elementa in cluster:
            lowerCamelCase__           =		_shared_dataset[elementa['base_index']]['content']
            for elementa in extremes:
                  lowerCamelCase__           =		_shared_dataset[elementa['base_index']]['content']
                  if jaccard_similarity(_a		,   _a							) >= jaccard_threshold:
                        elementa["copies"] += 1
                        break
            else:
                  lowerCamelCase__           =		1
                  extremes.append(_a							)
      return extremes



def   snake_case						(				_a:     Any		,   _a:     Tuple		,   _a:     Dict							)->		Union[str, Any]:
      '''simple docstring'''


      global _shared_dataset
      lowerCamelCase__           =		dataset
      lowerCamelCase__           =		[]
      lowerCamelCase__           =		partial(_find_cluster_extremes_shared		,   jaccard_threshold=_a							)
      with mp.Pool() as pool:
            for extremes in tqdm(
                pool.imap_unordered(
                    _a		,   _a		,   )		,   total=len(_a							)		,   ):
                  extremes_list.append(_a							)
      return extremes_list






def   snake_case						(				_a:     Type[Dataset]		,   _a:     float = 0.85							)->		Tuple[Type[Dataset], List[List[Dict]]]:
      '''simple docstring'''


      lowerCamelCase__           =		make_duplicate_clusters(_a		,   _a							)
      lowerCamelCase__           =		{x['base_index'] for cluster in duplicate_clusters for x in cluster}
      lowerCamelCase__           =		{}
      lowerCamelCase__           =		find_extremes(_a		,   _a		,   _a							)
      for extremes in extremes_clusters:
            for element in extremes:
                  lowerCamelCase__           =		element
      lowerCamelCase__           =		duplicate_indices - set(extreme_dict.keys()							)
      lowerCamelCase__           =		dataset.filter(lambda _a		,   _a							: idx not in remove_indices		,   with_indices=_a							)

      # update duplicate_clusters
      for cluster in duplicate_clusters:
            for element in cluster:
                  lowerCamelCase__           =		element['base_index'] in extreme_dict
                  if element["is_extreme"]:
                        lowerCamelCase__           =		extreme_dict[element['base_index']]['copies']

      print(F'Original dataset size: {len(_a							)}'							)
      print(F'Number of duplicate clusters: {len(_a							)}'							)
      print(F'Files in duplicate cluster: {len(_a							)}'							)
      print(F'Unique files in duplicate cluster: {len(_a							)}'							)
      print(F'Filtered dataset size: {len(_a							)}'							)

      return ds_filter, duplicate_clusters



