





'''simple docstring'''




from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase    )->     List[Any]:
	__UpperCAmelCase					=   []
	for part_id in partition_order:
		__UpperCAmelCase					=   df.where(F'SPARK_PARTITION_ID() = {part_id}'    ).collect()
		for row_idx, row in enumerate(_lowerCAmelCase    ):
			expected_row_ids_and_row_dicts.append((F'{part_id}_{row_idx}', row.asDict())    )
	return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def _lowerCAmelCase     (      )->     Any:
	__UpperCAmelCase					=   pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
	__UpperCAmelCase					=   spark.range(1_00    ).repartition(1    )
	__UpperCAmelCase					=   Spark(_lowerCAmelCase    )
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16    )
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def _lowerCAmelCase     (      )->     List[str]:
	__UpperCAmelCase					=   pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
	__UpperCAmelCase					=   spark.range(10    ).repartition(2    )
	__UpperCAmelCase					=   [1, 0]
	__UpperCAmelCase					=   _generate_iterable_examples(_lowerCAmelCase       ,    _lowerCAmelCase    )  # Reverse the partitions.
	__UpperCAmelCase					=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    _lowerCAmelCase    )

	for i, (row_id, row_dict) in enumerate(generate_fn()    ):
		__UpperCAmelCase					,    __UpperCAmelCase					=   expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def _lowerCAmelCase     (      )->     Any:
	__UpperCAmelCase					=   pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
	__UpperCAmelCase					=   spark.range(10    ).repartition(1    )
	__UpperCAmelCase					=   SparkExamplesIterable(_lowerCAmelCase    )
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(_lowerCAmelCase    ):
		assert row_id == F'0_{i}'
		assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def _lowerCAmelCase     (      )->     Optional[int]:
	__UpperCAmelCase					=   pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
	__UpperCAmelCase					=   spark.range(30    ).repartition(3    )
	# Mock the generator so that shuffle reverses the partition indices.
	with patch('numpy.random.Generator'    ) as generator_mock:
		__UpperCAmelCase					=   lambda _lowerCAmelCase    : x.reverse()
		__UpperCAmelCase					=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    [2, 1, 0]    )

		__UpperCAmelCase					=   SparkExamplesIterable(_lowerCAmelCase    ).shuffle_data_sources(_lowerCAmelCase    )
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(_lowerCAmelCase    ):
			__UpperCAmelCase					,    __UpperCAmelCase					=   expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def _lowerCAmelCase     (      )->     List[str]:
	__UpperCAmelCase					=   pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
	__UpperCAmelCase					=   spark.range(20    ).repartition(4    )

	# Partitions 0 and 2
	__UpperCAmelCase					=   SparkExamplesIterable(_lowerCAmelCase    ).shard_data_sources(worker_id=0       ,    num_workers=2    )
	assert shard_it_a.n_shards == 2
	__UpperCAmelCase					=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    [0, 2]    )
	for i, (row_id, row_dict) in enumerate(_lowerCAmelCase    ):
		__UpperCAmelCase					,    __UpperCAmelCase					=   expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	__UpperCAmelCase					=   SparkExamplesIterable(_lowerCAmelCase    ).shard_data_sources(worker_id=1       ,    num_workers=2    )
	assert shard_it_a.n_shards == 2
	__UpperCAmelCase					=   _get_expected_row_ids_and_row_dicts_for_partition_order(_lowerCAmelCase       ,    [1, 3]    )
	for i, (row_id, row_dict) in enumerate(_lowerCAmelCase    ):
		__UpperCAmelCase					,    __UpperCAmelCase					=   expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def _lowerCAmelCase     (      )->     Optional[Any]:
	__UpperCAmelCase					=   pyspark.sql.SparkSession.builder.master('local[*]'    ).appName('pyspark'    ).getOrCreate()
	__UpperCAmelCase					=   spark.range(1_00    ).repartition(1    )
	__UpperCAmelCase					=   Spark(_lowerCAmelCase    )
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1    )
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_00












'''simple docstring'''




import math
from typing import Callable, List, Optional, Union

import numpy as np
import PIL
import torch
from PIL import Image
from transformers import CLIPTextModel, CLIPTokenizer

from diffusers.models import AutoencoderKL, UNetaDConditionModel
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale import StableDiffusionUpscalePipeline
from diffusers.schedulers import DDIMScheduler, DDPMScheduler, LMSDiscreteScheduler, PNDMScheduler







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase=[]    )->     Any:
 __UpperCAmelCase					=   size[0] - overlap_pixels * 2
 __UpperCAmelCase					=   size[1] - overlap_pixels * 2
 for letter in ["l", "r"]:
  if letter in remove_borders:
   size_x += overlap_pixels
 for letter in ["t", "b"]:
  if letter in remove_borders:
   size_y += overlap_pixels
 __UpperCAmelCase					=   np.ones((size_y, size_x)       ,    dtype=np.uinta    ) * 2_55
 __UpperCAmelCase					=   np.pad(_lowerCAmelCase       ,    mode='linear_ramp'       ,    pad_width=_lowerCAmelCase       ,    end_values=0    )

 if "l" in remove_borders:
  __UpperCAmelCase					=   mask[:, overlap_pixels : mask.shape[1]]
 if "r" in remove_borders:
  __UpperCAmelCase					=   mask[:, 0 : mask.shape[1] - overlap_pixels]
 if "t" in remove_borders:
  __UpperCAmelCase					=   mask[overlap_pixels : mask.shape[0], :]
 if "b" in remove_borders:
  __UpperCAmelCase					=   mask[0 : mask.shape[0] - overlap_pixels, :]
 return mask







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase    )->     Any:
 return max(_lowerCAmelCase       ,    min(_lowerCAmelCase       ,    _lowerCAmelCase    )    )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase    )->     Dict:
 return (
     clamp(rect[0]       ,    min[0]       ,    max[0]    ),
     clamp(rect[1]       ,    min[1]       ,    max[1]    ),
     clamp(rect[2]       ,    min[0]       ,    max[0]    ),
     clamp(rect[3]       ,    min[1]       ,    max[1]    ),
 )







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase    )->     Dict:
 __UpperCAmelCase					=   list(_lowerCAmelCase    )
 rect[0] -= overlap
 rect[1] -= overlap
 rect[2] += overlap
 rect[3] += overlap
 __UpperCAmelCase					=   clamp_rect(_lowerCAmelCase       ,    [0, 0]       ,    [image_size[0], image_size[1]]    )
 return rect







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase       ,    _lowerCAmelCase    )->     Tuple:
 __UpperCAmelCase					=   Image.new('RGB'       ,    (tile.size[0] + original_slice, tile.size[1])    )
 result.paste(
     original_image.resize((tile.size[0], tile.size[1])       ,    Image.BICUBIC    ).crop(
         (slice_x, 0, slice_x + original_slice, tile.size[1])    )       ,    (0, 0)       ,    )
 result.paste(_lowerCAmelCase       ,    (original_slice, 0)    )
 return result







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase    )->     List[Any]:
 __UpperCAmelCase					=   (original_image_slice * 4, 0, tile.size[0], tile.size[1])
 __UpperCAmelCase					=   tile.crop(_lowerCAmelCase    )
 return tile







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase    )->     Optional[int]:
 __UpperCAmelCase					=   n % d
 return n - divisor






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A = 350     ,			):
     super().__init__(
         vae=__A     ,			text_encoder=__A     ,			tokenizer=__A     ,			unet=__A     ,			low_res_scheduler=__A     ,			scheduler=__A     ,			max_noise_level=__A     ,			)



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A     ,			**__A					):
     torch.manual_seed(0					)
     __UpperCAmelCase					=   (
         min(image.size[0] - (tile_size + original_image_slice)     ,			x * tile_size					),
         min(image.size[1] - (tile_size + original_image_slice)     ,			y * tile_size					),
         min(image.size[0]     ,			(x + 1) * tile_size					),
         min(image.size[1]     ,			(y + 1) * tile_size					),
     )
     __UpperCAmelCase					=   add_overlap_rect(__A     ,			__A     ,			image.size					)
     __UpperCAmelCase					=   image.crop(__A					)
     __UpperCAmelCase					=   ((crop_rect[0] + ((crop_rect[2] - crop_rect[0]) / 2)) / image.size[0]) * tile.size[0]
     __UpperCAmelCase					=   translated_slice_x - (original_image_slice / 2)
     __UpperCAmelCase					=   max(0     ,			__A					)
     __UpperCAmelCase					=   squeeze_tile(__A     ,			__A     ,			__A     ,			__A					)
     __UpperCAmelCase					=   to_input.size
     __UpperCAmelCase					=   to_input.resize((tile_size, tile_size)     ,			Image.BICUBIC					)
     __UpperCAmelCase					=   super(__A     ,			self					).__call__(image=__A     ,			**__A					).images[0]
     __UpperCAmelCase					=   upscaled_tile.resize((orig_input_size[0] * 4, orig_input_size[1] * 4)     ,			Image.BICUBIC					)
     __UpperCAmelCase					=   unsqueeze_tile(__A     ,			__A					)
     __UpperCAmelCase					=   upscaled_tile.resize((tile.size[0] * 4, tile.size[1] * 4)     ,			Image.BICUBIC					)
     __UpperCAmelCase					=   []
     if x == 0:
      remove_borders.append('l'					)
     elif crop_rect[2] == image.size[0]:
      remove_borders.append('r'					)
     if y == 0:
      remove_borders.append('t'					)
     elif crop_rect[3] == image.size[1]:
      remove_borders.append('b'					)
     __UpperCAmelCase					=   Image.fromarray(
         make_transparency_mask(
             (upscaled_tile.size[0], upscaled_tile.size[1])     ,			tile_border * 4     ,			remove_borders=__A					)     ,			mode='L'     ,			)
     final_image.paste(
         __A     ,			(crop_rect_with_overlap[0] * 4, crop_rect_with_overlap[1] * 4)     ,			__A					)




    @torch.no_grad()
    def __call__(  self     ,			__A     ,			__A     ,			__A = 75     ,			__A = 9.0     ,			__A = 50     ,			__A = None     ,			__A = 1     ,			__A = 0.0     ,			__A = None     ,			__A = None     ,			__A = None     ,			__A = 1     ,			__A = 128     ,			__A = 32     ,			__A = 32     ,			):
     __UpperCAmelCase					=   Image.new('RGB'     ,			(image.size[0] * 4, image.size[1] * 4)					)
     __UpperCAmelCase					=   math.ceil(image.size[0] / tile_size					)
     __UpperCAmelCase					=   math.ceil(image.size[1] / tile_size					)
     __UpperCAmelCase					=   tcx * tcy
     __UpperCAmelCase					=   0
     for y in range(__A					):
      for x in range(__A					):
       self._process_tile(
           __A     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A     ,			__A     ,			prompt=__A     ,			num_inference_steps=__A     ,			guidance_scale=__A     ,			noise_level=__A     ,			negative_prompt=__A     ,			num_images_per_prompt=__A     ,			eta=__A     ,			generator=__A     ,			latents=__A     ,			)
       current_count += 1
       if callback is not None:
        callback({'progress': current_count / total_tile_count, 'image': final_image}					)
     return final_image







def _lowerCAmelCase     (      )->     str:
 # Run a demo
 __UpperCAmelCase					=   'stabilityai/stable-diffusion-x4-upscaler'
 __UpperCAmelCase					=   StableDiffusionTiledUpscalePipeline.from_pretrained(_lowerCAmelCase       ,    revision='fp16'       ,    torch_dtype=torch.floataa    )
 __UpperCAmelCase					=   pipe.to('cuda'    )
 __UpperCAmelCase					=   Image.open('../../docs/source/imgs/diffusers_library.jpg'    )

 def callback(_lowerCAmelCase    ):
  print(F'progress: {obj["progress"]:.4f}'    )
  obj["image"].save('diffusers_library_progress.jpg'    )

 __UpperCAmelCase					=   pipe(image=_lowerCAmelCase       ,    prompt='Black font, white background, vector'       ,    noise_level=40       ,    callback=_lowerCAmelCase    )
 final_image.save('diffusers_library.jpg'    )


if __name__ == "__main__":
    main()





