


import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil







class  snake_case_     (lowerCamelCase_   ):
					def 				lowerCamelCase__(   self     :Union[str, Any]   )      ->					Any:
												a__    =  pa.array(TypedSequence([1, 2, 3]   )   )
												self.assertEqual(arr.type    ,pa.intaa()   )
					def 				lowerCamelCase__(   self     :Dict   )      ->					List[Any]:
												with self.assertRaises(__snake_case   ):
																			a__    =  pa.array(TypedSequence([1, 2, 3]   )    ,type=pa.intaa()   )
					def 				lowerCamelCase__(   self     :List[str]   )      ->					List[str]:
												with self.assertRaises(__snake_case   ):
																			a__    =  pa.array(TypedSequence([1, 2, 3]    ,try_type=Value('bool'   )    ,type=Value('int64'   )   )   )
					def 				lowerCamelCase__(   self     :Union[str, Any]   )      ->					Any:
												a__    =  pa.array(TypedSequence([1, 2, 3]    ,type=Value('int32'   )   )   )
												self.assertEqual(arr.type    ,pa.intaa()   )
					def 				lowerCamelCase__(   self     :List[Any]   )      ->					Any:
												with self.assertRaises((TypeError, pa.lib.ArrowInvalid)   ):
																			a__    =  pa.array(TypedSequence(['foo', 'bar']    ,type=Value('int64'   )   )   )
					def 				lowerCamelCase__(   self     :Optional[int]   )      ->					Optional[int]:
												a__    =  pa.array(TypedSequence([1, 2, 3]    ,try_type=Value('int32'   )   )   )
												self.assertEqual(arr.type    ,pa.intaa()   )
					def 				lowerCamelCase__(   self     :List[str]   )      ->					str:
												a__    =  pa.array(TypedSequence(['foo', 'bar']    ,try_type=Value('int64'   )   )   )
												self.assertEqual(arr.type    ,pa.string()   )
					def 				lowerCamelCase__(   self     :Tuple   )      ->					List[str]:
												a__    =  pa.array(TypedSequence([[[1, 2, 3]]]    ,type=ArrayaD((1, 3)    ,'int64'   )   )   )
												self.assertEqual(arr.type    ,ArrayaDExtensionType((1, 3)    ,'int64'   )   )
					def 				lowerCamelCase__(   self     :List[Any]   )      ->					Optional[Any]:
												with self.assertRaises((TypeError, pa.lib.ArrowInvalid)   ):
																			a__    =  pa.array(TypedSequence(['foo', 'bar']    ,type=ArrayaD((1, 3)    ,'int64'   )   )   )
					def 				lowerCamelCase__(   self     :Dict   )      ->					Tuple:
												a__    =  pa.array(TypedSequence([[[1, 2, 3]]]    ,try_type=ArrayaD((1, 3)    ,'int64'   )   )   )
												self.assertEqual(arr.type    ,ArrayaDExtensionType((1, 3)    ,'int64'   )   )
					def 				lowerCamelCase__(   self     :Tuple   )      ->					Optional[Any]:
												a__    =  pa.array(TypedSequence(['foo', 'bar']    ,try_type=ArrayaD((1, 3)    ,'int64'   )   )   )
												self.assertEqual(arr.type    ,pa.string()   )



					@require_pil
					def 				lowerCamelCase__(   self     :Optional[int]   )      ->					str:
												import PIL.Image

												a__    =  PIL.Image.fromarray(np.arange(10    ,dtype=np.uinta   ).reshape(2    ,5   )   )
												with patch(
												    'datasets.arrow_writer.cast_to_python_objects'    ,side_effect=__snake_case   ) as mock_cast_to_python_objects:
																			a__    =  pa.array(TypedSequence([{'path': None, 'bytes': B'image_bytes'}, pil_image]    ,type=Image()   )   )
																			a__   ,      a__    =  mock_cast_to_python_objects.call_args_list[-1]
																			self.assertIn('optimize_list_casting'    ,__snake_case   )
																			self.assertFalse(kwargs['optimize_list_casting']   )






def 							__lowercase     (				__lowerCAmelCase							:       Optional[Any]      ,					__lowerCAmelCase							:       int     ):
							a__    =  pa.BufferReader(__lowerCAmelCase     ) if isinstance(__lowerCAmelCase      ,					pa.Buffer     ) else pa.memory_map(__lowerCAmelCase     )
							a__    =  pa.ipc.open_stream(__lowerCAmelCase     )
							a__    =  f.read_all()
							assert len(pa_table.to_batches()     ) == expected_num_chunks
							assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
							del pa_table






@pytest.mark.parametrize('writer_batch_size'      ,					[None, 1, 1_0]     )
@pytest.mark.parametrize(
    'fields'      ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def 							__lowercase     (				__lowerCAmelCase							:       Any      ,					__lowerCAmelCase							:       Union[str, Any]     ):
							a__    =  pa.BufferOutputStream()
							a__    =  pa.schema(__lowerCAmelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCAmelCase      ,					schema=__lowerCAmelCase      ,					writer_batch_size=__lowerCAmelCase     ) as writer:
														writer.write({'col_1': 'foo', 'col_2': 1}     )
														writer.write({'col_1': 'bar', 'col_2': 2}     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														a__    =  {'col_1': pa.string(), 'col_2': pa.intaa()}
							assert writer._schema == pa.schema(__lowerCAmelCase      ,					metadata=writer._schema.metadata     )
							_check_output(output.getvalue()      ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )






def 							__lowercase     (				):
							a__    =  pa.BufferOutputStream()
							a__    =  Features({'labels': ClassLabel(names=['neg', 'pos']     )}     )
							with ArrowWriter(stream=__lowerCAmelCase      ,					features=__lowerCAmelCase     ) as writer:
														writer.write({'labels': 0}     )
														writer.write({'labels': 1}     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							assert writer._schema == features.arrow_schema
							assert writer._schema.metadata == features.arrow_schema.metadata
							a__    =  pa.BufferReader(output.getvalue()     )
							a__    =  pa.ipc.open_stream(__lowerCAmelCase     )
							a__    =  f.read_all()
							a__    =  pa_table.schema
							assert pa_table.num_rows == 2
							assert schema == features.arrow_schema
							assert schema.metadata == features.arrow_schema.metadata
							assert features == Features.from_arrow_schema(__lowerCAmelCase     )






@pytest.mark.parametrize('writer_batch_size'      ,					[None, 1, 1_0]     )
def 							__lowercase     (				__lowerCAmelCase							:       Dict     ):
							a__    =  pa.BufferOutputStream()
							with ArrowWriter(
							    stream=__lowerCAmelCase      ,					writer_batch_size=__lowerCAmelCase      ,					hash_salt='split_name'      ,					check_duplicates=__lowerCAmelCase      ,					) as writer:
														with pytest.raises(__lowerCAmelCase     ):
																					writer.write({'col_1': 'foo', 'col_2': 1}      ,					key=[1, 2]     )
																					a__   ,      a__    =  writer.finalize()






@pytest.mark.parametrize('writer_batch_size'      ,					[None, 2, 1_0]     )
def 							__lowercase     (				__lowerCAmelCase							:       List[Any]     ):
							a__    =  pa.BufferOutputStream()
							with ArrowWriter(
							    stream=__lowerCAmelCase      ,					writer_batch_size=__lowerCAmelCase      ,					hash_salt='split_name'      ,					check_duplicates=__lowerCAmelCase      ,					) as writer:
														with pytest.raises(__lowerCAmelCase     ):
																					writer.write({'col_1': 'foo', 'col_2': 1}      ,					key=1_0     )
																					writer.write({'col_1': 'bar', 'col_2': 2}      ,					key=1_0     )
																					a__   ,      a__    =  writer.finalize()






@pytest.mark.parametrize('writer_batch_size'      ,					[None, 2, 1_0]     )
def 							__lowercase     (				__lowerCAmelCase							:       Optional[Any]     ):
							a__    =  pa.BufferOutputStream()
							with ArrowWriter(
							    stream=__lowerCAmelCase      ,					writer_batch_size=__lowerCAmelCase      ,					hash_salt='split_name'      ,					check_duplicates=__lowerCAmelCase      ,					) as writer:
														writer.write({'col_1': 'foo', 'col_2': 1}      ,					key=1     )
														writer.write({'col_1': 'bar', 'col_2': 2}      ,					key=2     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							_check_output(output.getvalue()      ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )






@pytest.mark.parametrize('writer_batch_size'      ,					[None, 1, 1_0]     )
@pytest.mark.parametrize(
    'fields'      ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def 							__lowercase     (				__lowerCAmelCase							:       str      ,					__lowerCAmelCase							:       Dict     ):
							a__    =  pa.BufferOutputStream()
							a__    =  pa.schema(__lowerCAmelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCAmelCase      ,					schema=__lowerCAmelCase      ,					writer_batch_size=__lowerCAmelCase     ) as writer:
														writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}     )
														writer.write_batch({'col_1': [], 'col_2': []}     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														a__    =  {'col_1': pa.string(), 'col_2': pa.intaa()}
							assert writer._schema == pa.schema(__lowerCAmelCase      ,					metadata=writer._schema.metadata     )
							_check_output(output.getvalue()      ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )






@pytest.mark.parametrize('writer_batch_size'      ,					[None, 1, 1_0]     )
@pytest.mark.parametrize(
    'fields'      ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def 							__lowercase     (				__lowerCAmelCase							:       Dict      ,					__lowerCAmelCase							:       Union[str, Any]     ):
							a__    =  pa.BufferOutputStream()
							a__    =  pa.schema(__lowerCAmelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCAmelCase      ,					schema=__lowerCAmelCase      ,					writer_batch_size=__lowerCAmelCase     ) as writer:
														writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}     )     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														a__    =  {'col_1': pa.string(), 'col_2': pa.intaa()}
							assert writer._schema == pa.schema(__lowerCAmelCase      ,					metadata=writer._schema.metadata     )
							_check_output(output.getvalue()      ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )






@pytest.mark.parametrize('writer_batch_size'      ,					[None, 1, 1_0]     )
@pytest.mark.parametrize(
    'fields'      ,					[None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]     )
def 							__lowercase     (				__lowerCAmelCase							:       Any      ,					__lowerCAmelCase							:       Dict     ):
							a__    =  pa.BufferOutputStream()
							a__    =  pa.schema(__lowerCAmelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCAmelCase      ,					schema=__lowerCAmelCase      ,					writer_batch_size=__lowerCAmelCase     ) as writer:
														writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}     )     )
														writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}     )     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														a__    =  {'col_1': pa.string(), 'col_2': pa.intaa()}
							assert writer._schema == pa.schema(__lowerCAmelCase      ,					metadata=writer._schema.metadata     )
							_check_output(output.getvalue()      ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )






def 							__lowercase     (				):
							with tempfile.TemporaryDirectory() as tmp_dir:
														a__    =  {'col_1': pa.string(), 'col_2': pa.intaa()}
														a__    =  os.path.join(__lowerCAmelCase      ,					'test.arrow'     )
														with ArrowWriter(path=__lowerCAmelCase      ,					schema=pa.schema(__lowerCAmelCase     )     ) as writer:
																					writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}     )
																					a__   ,      a__    =  writer.finalize()
														assert num_examples == 2
														assert num_bytes > 0
														assert writer._schema == pa.schema(__lowerCAmelCase      ,					metadata=writer._schema.metadata     )
														_check_output(__lowerCAmelCase      ,					1     )






def 							__lowercase     (				__lowerCAmelCase							:       Tuple     ):
							if pa.types.is_list(__lowerCAmelCase     ):
														return get_base_dtype(arr_type.value_type     )
							else:
														return arr_type






def 							__lowercase     (				__lowerCAmelCase							:       Optional[Any]      ,					__lowerCAmelCase							:       str     ):
							if isinstance(lst[0]      ,					__lowerCAmelCase     ):
														change_first_primitive_element_in_list(lst[0]      ,					__lowerCAmelCase     )
							else:
														a__    =  value






@pytest.mark.parametrize('optimized_int_type, expected_dtype'      ,					[(None, pa.intaa()), (Value('int32'     ), pa.intaa())]     )
@pytest.mark.parametrize('sequence'      ,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]     )
def 							__lowercase     (				__lowerCAmelCase							:       int      ,					__lowerCAmelCase							:       Optional[int]      ,					__lowerCAmelCase							:       str     ):
							a__    =  pa.array(TypedSequence(__lowerCAmelCase      ,					optimized_int_type=__lowerCAmelCase     )     )
							assert get_base_dtype(arr.type     ) == expected_dtype






@pytest.mark.parametrize(
    'col, expected_dtype'      ,					[
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]      ,					)
@pytest.mark.parametrize('sequence'      ,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]     )
def 							__lowercase     (				__lowerCAmelCase							:       Any      ,					__lowerCAmelCase							:       str      ,					__lowerCAmelCase							:       str     ):
							# in range
							a__    =  pa.array(OptimizedTypedSequence(__lowerCAmelCase      ,					col=__lowerCAmelCase     )     )
							assert get_base_dtype(arr.type     ) == expected_dtype

							# not in range
							if col != "other":
														# avoids errors due to in-place modifications
														a__    =  copy.deepcopy(__lowerCAmelCase     )
														a__    =  np.iinfo(expected_dtype.to_pandas_dtype()     ).max + 1
														change_first_primitive_element_in_list(__lowerCAmelCase      ,					__lowerCAmelCase     )
														a__    =  pa.array(OptimizedTypedSequence(__lowerCAmelCase      ,					col=__lowerCAmelCase     )     )
														assert get_base_dtype(arr.type     ) == pa.intaa()






@pytest.mark.parametrize('raise_exception'      ,					[False, True]     )
def 							__lowercase     (				__lowerCAmelCase							:       str      ,					__lowerCAmelCase							:       List[Any]     ):
							a__    =  str(tmp_path / 'dataset-train.arrow'     )
							try:
														with ArrowWriter(path=__lowerCAmelCase     ) as writer:
																					if raise_exception:
																												raise pa.lib.ArrowInvalid()
																					else:
																												writer.stream.close()
							except pa.lib.ArrowInvalid:
														pass
							finally:
														assert writer.stream.closed






def 							__lowercase     (				__lowerCAmelCase							:       Optional[Any]     ):
							a__    =  'mock://dataset-train.arrow'
							with ArrowWriter(path=__lowerCAmelCase      ,					storage_options=mockfs.storage_options     ) as writer:
														assert isinstance(writer._fs      ,					type(__lowerCAmelCase     )     )
														assert writer._fs.storage_options == mockfs.storage_options
														writer.write({'col_1': 'foo', 'col_2': 1}     )
														writer.write({'col_1': 'bar', 'col_2': 2}     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							assert mockfs.exists(__lowerCAmelCase     )






def 							__lowercase     (				):
							a__    =  pa.BufferOutputStream()
							with ParquetWriter(stream=__lowerCAmelCase     ) as writer:
														writer.write({'col_1': 'foo', 'col_2': 1}     )
														writer.write({'col_1': 'bar', 'col_2': 2}     )
														a__   ,      a__    =  writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							a__    =  pa.BufferReader(output.getvalue()     )
							a__    =  pq.read_table(__lowerCAmelCase     )
							assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}






@require_pil
@pytest.mark.parametrize('embed_local_files'      ,					[False, True]     )
def 							__lowercase     (				__lowerCAmelCase							:       Union[str, Any]      ,					__lowerCAmelCase							:       int     ):
							import PIL.Image

							a__    =  str(tmp_path / 'test_image_rgb.jpg'     )
							PIL.Image.fromarray(np.zeros((5, 5)      ,					dtype=np.uinta     )     ).save(__lowerCAmelCase      ,					format='png'     )
							a__    =  pa.BufferOutputStream()
							with ParquetWriter(
							    stream=__lowerCAmelCase      ,					features=Features({'image': Image()}     )      ,					embed_local_files=__lowerCAmelCase     ) as writer:
														writer.write({'image': image_path}     )
														writer.finalize()
							a__    =  pa.BufferReader(output.getvalue()     )
							a__    =  pq.read_table(__lowerCAmelCase     )
							a__    =  pa_table.to_pydict()
							if embed_local_files:
														assert isinstance(out['image'][0]['path']      ,					__lowerCAmelCase     )
														with open(__lowerCAmelCase      ,					'rb'     ) as f:
																					assert out["image"][0]["bytes"] == f.read()
							else:
														assert out["image"][0]["path"] == image_path
														assert out["image"][0]["bytes"] is None






def 							__lowercase     (				):
							a__    =  pa.schema([pa.field('col_1'      ,					pa.string()      ,					nullable=__lowerCAmelCase     )]     )
							a__    =  pa.BufferOutputStream()
							with ArrowWriter(stream=__lowerCAmelCase     ) as writer:
														writer._build_writer(inferred_schema=__lowerCAmelCase     )
														assert writer._schema == pa.schema([pa.field('col_1'      ,					pa.string()     )]     )








from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case							: Optional[Any]      =   {'''configuration_focalnet''': ['''FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FocalNetConfig''']}


try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						snake_case							: Optional[int]      =   [
						    '''FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''FocalNetForImageClassification''',
						    '''FocalNetForMaskedImageModeling''',
						    '''FocalNetBackbone''',
						    '''FocalNetModel''',
						    '''FocalNetPreTrainedModel''',
						]

if TYPE_CHECKING:
						from .configuration_focalnet import FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP, FocalNetConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_focalnet import (
												    FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST,
												    FocalNetBackbone,
												    FocalNetForImageClassification,
												    FocalNetForMaskedImageModeling,
												    FocalNetModel,
												    FocalNetPreTrainedModel,
												)

else:
						import sys

						snake_case							: Union[str, Any]      =   _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




