



'''simple docstring'''




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam

class     UpperCamelCase_				(							datasets.BeamBasedBuilder    ):





	def 				_lowercase(						self					)    ->    Optional[Any]:
								return datasets.DatasetInfo(
								    features=datasets.Features({"""content""": datasets.Value("""string"""					)}					)	,	supervised_keys=A	,	)





	def 				_lowercase(						self	,	A	,	A					)    ->    Dict:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN	,	gen_kwargs={"""examples""": get_test_dummy_examples()}					)]






	def 				_lowercase(						self	,	A	,	A					)    ->    List[str]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(A					)




class     UpperCamelCase_				(							datasets.BeamBasedBuilder    ):





	def 				_lowercase(						self					)    ->    Dict:
								return datasets.DatasetInfo(
								    features=datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""					)}					)}					)	,	supervised_keys=A	,	)





	def 				_lowercase(						self	,	A	,	A					)    ->    Optional[Any]:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN	,	gen_kwargs={"""examples""": get_test_nested_examples()}					)
								]






	def 				_lowercase(						self	,	A	,	A					)    ->    Optional[int]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(A					)




def    __lowerCamelCase						(      )   ->    Any:
							return [(i, {"content": content}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]	)]



def    __lowerCamelCase						(      )   ->    int:
							return [(i, {"a": {"b": [content]}}) for i, content in enumerate(["""foo""", """bar""", """foobar"""]	)]


class     UpperCamelCase_				(							__magic_name__    ):





	@require_beam
	def 				_lowercase(						self					)    ->    List[str]:
								UpperCAmelCase  :	int							  =	len(get_test_dummy_examples()					)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase  :	Any							  =	DummyBeamDataset(cache_dir=A	,	beam_runner="""DirectRunner"""					)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(A	,	builder.name	,	"""default"""	,	"""0.0.0"""	,	f'''{builder.name}-train.arrow'''					)					)					)
															self.assertDictEqual(builder.info.features	,	datasets.Features({"""content""": datasets.Value("""string"""					)}					)					)
															UpperCAmelCase  :	List[str]							  =	builder.as_dataset()
															self.assertEqual(dset["""train"""].num_rows	,	A					)
															self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples	,	A					)
															self.assertDictEqual(dset["""train"""][0]	,	get_test_dummy_examples()[0][1]					)
															self.assertDictEqual(
															    dset["""train"""][expected_num_examples - 1]	,	get_test_dummy_examples()[expected_num_examples - 1][1]					)
															self.assertTrue(
															    os.path.exists(os.path.join(A	,	builder.name	,	"""default"""	,	"""0.0.0"""	,	"""dataset_info.json"""					)					)					)
															del dset





	@require_beam
	def 				_lowercase(						self					)    ->    Any:
								import apache_beam as beam

								UpperCAmelCase  :	List[str]							  =	beam.io.parquetio.WriteToParquet

								UpperCAmelCase  :	Union[str, Any]							  =	len(get_test_dummy_examples()					)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase  :	Union[str, Any]							  =	DummyBeamDataset(cache_dir=A	,	beam_runner="""DirectRunner"""					)
															with patch("""apache_beam.io.parquetio.WriteToParquet"""					) as write_parquet_mock:
																						UpperCAmelCase  :	List[Any]							  =	partial(A	,	num_shards=2					)
																						builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            A	,	builder.name	,	"""default"""	,	"""0.0.0"""	,	f'''{builder.name}-train-00000-of-00002.arrow'''					)					)					)
															self.assertTrue(
															    os.path.exists(
															        os.path.join(
															            A	,	builder.name	,	"""default"""	,	"""0.0.0"""	,	f'''{builder.name}-train-00000-of-00002.arrow'''					)					)					)
															self.assertDictEqual(builder.info.features	,	datasets.Features({"""content""": datasets.Value("""string"""					)}					)					)
															UpperCAmelCase  :	Dict							  =	builder.as_dataset()
															self.assertEqual(dset["""train"""].num_rows	,	A					)
															self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples	,	A					)
															# Order is not preserved when sharding, so we just check that all the elements are there
															self.assertListEqual(sorted(dset["""train"""]["""content"""]					)	,	sorted(["""foo""", """bar""", """foobar"""]					)					)
															self.assertTrue(
															    os.path.exists(os.path.join(A	,	builder.name	,	"""default"""	,	"""0.0.0"""	,	"""dataset_info.json"""					)					)					)
															del dset





	@require_beam
	def 				_lowercase(						self					)    ->    List[str]:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase  :	Tuple							  =	DummyBeamDataset(cache_dir=A					)
															self.assertRaises(datasets.builder.MissingBeamOptions	,	builder.download_and_prepare					)






	@require_beam
	def 				_lowercase(						self					)    ->    int:
								UpperCAmelCase  :	Tuple							  =	len(get_test_nested_examples()					)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
															UpperCAmelCase  :	Any							  =	NestedBeamDataset(cache_dir=A	,	beam_runner="""DirectRunner"""					)
															builder.download_and_prepare()
															self.assertTrue(
															    os.path.exists(
															        os.path.join(A	,	builder.name	,	"""default"""	,	"""0.0.0"""	,	f'''{builder.name}-train.arrow'''					)					)					)
															self.assertDictEqual(
															    builder.info.features	,	datasets.Features({"""a""": datasets.Sequence({"""b""": datasets.Value("""string"""					)}					)}					)					)
															UpperCAmelCase  :	Optional[Any]							  =	builder.as_dataset()
															self.assertEqual(dset["""train"""].num_rows	,	A					)
															self.assertEqual(dset["""train"""].info.splits["""train"""].num_examples	,	A					)
															self.assertDictEqual(dset["""train"""][0]	,	get_test_nested_examples()[0][1]					)
															self.assertDictEqual(
															    dset["""train"""][expected_num_examples - 1]	,	get_test_nested_examples()[expected_num_examples - 1][1]					)
															self.assertTrue(
															    os.path.exists(os.path.join(A	,	builder.name	,	"""default"""	,	"""0.0.0"""	,	"""dataset_info.json"""					)					)					)
															del dset






'''simple docstring'''




# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
							import jax
							import jaxlib

a     :    Any      						=					get_logger()

a     :    Optional[dict]      						=					None

class     UpperCamelCase_				(							TensorFormatter[Mapping, 'jax.Array', Mapping]    ):





	def __init__(						self	,	A=None	,	A=None	,	**A					)    ->    str:
								super().__init__(features=A					)
								import jax
								from jaxlib.xla_client import Device

								if isinstance(A	,	A					):
															raise ValueError(
															    f'''Expected {device} to be a `str` not {type(A					)}, as `jaxlib.xla_extension.Device` '''
															    """is not serializable neither with `pickle` nor with `dill`. Instead you can surround """
															    """the device with `str()` to get its string identifier that will be internally mapped """
															    """to the actual `jaxlib.xla_extension.Device`."""					)
								UpperCAmelCase  :	Optional[int]							  =	device if isinstance(A	,	A					) else str(jax.devices()[0]					)
								# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
								# with `pickle` nor with `dill`, so we need to use a global variable instead
								global DEVICE_MAPPING
								if DEVICE_MAPPING is None:
															UpperCAmelCase  :	Any							  =	self._map_devices_to_str()
								if self.device not in list(DEVICE_MAPPING.keys()					):
															logger.warning(
															    f'''Device with string identifier {self.device} not listed among the available '''
															    f'''devices: {list(DEVICE_MAPPING.keys()					)}, so falling back to the default '''
															    f'''device: {str(jax.devices()[0]					)}.'''					)
															UpperCAmelCase  :	List[Any]							  =	str(jax.devices()[0]					)
								UpperCAmelCase  :	Union[str, Any]							  =	jnp_array_kwargs





	@staticmethod
	def 				_lowercase(						)    ->    Dict[str, "jaxlib.xla_extension.Device"]:
								import jax

								return {str(A					): device for device in jax.devices()}





	def 				_lowercase(						self	,	A					)    ->    str:
								import jax
								import jax.numpy as jnp

								if isinstance(A	,	A					) and column:
															if all(
															    isinstance(A	,	jax.Array					) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column					):
																						return jnp.stack(A	,	axis=0					)
								return column





	def 				_lowercase(						self	,	A					)    ->    Tuple:
								import jax
								import jax.numpy as jnp

								if isinstance(A	,	(str, bytes, type(A					))					):
															return value
								elif isinstance(A	,	(np.character, np.ndarray)					) and np.issubdtype(value.dtype	,	np.character					):
															return value.tolist()

								UpperCAmelCase  :	List[str]							  =	{}

								if isinstance(A	,	(np.number, np.ndarray)					) and np.issubdtype(value.dtype	,	np.integer					):
															# the default int precision depends on the jax config
															# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
															if jax.config.jax_enable_xaa:
																						UpperCAmelCase  :	str							  =	{"""dtype""": jnp.intaa}
															else:
																						UpperCAmelCase  :	int							  =	{"""dtype""": jnp.intaa}
								elif isinstance(A	,	(np.number, np.ndarray)					) and np.issubdtype(value.dtype	,	np.floating					):
															UpperCAmelCase  :	Any							  =	{"""dtype""": jnp.floataa}
								elif config.PIL_AVAILABLE and "PIL" in sys.modules:
															import PIL.Image

															if isinstance(A	,	PIL.Image.Image					):
																						UpperCAmelCase  :	List[str]							  =	np.asarray(A					)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
								global DEVICE_MAPPING
								if DEVICE_MAPPING is None:
															UpperCAmelCase  :	Dict							  =	self._map_devices_to_str()

								with jax.default_device(DEVICE_MAPPING[self.device]					):
															# calling jnp.array on a np.ndarray does copy the data
															# see https://github.com/google/jax/issues/4486
															return jnp.array(A	,	**{**default_dtype, **self.jnp_array_kwargs}					)





	def 				_lowercase(						self	,	A					)    ->    Tuple:
								import jax

								# support for torch, tf, jax etc.
								if config.TORCH_AVAILABLE and "torch" in sys.modules:
															import torch

															if isinstance(A	,	torch.Tensor					):
																						return self._tensorize(data_struct.detach().cpu().numpy()[()]					)
								if hasattr(A	,	"""__array__"""					) and not isinstance(A	,	jax.Array					):
															UpperCAmelCase  :	Optional[int]							  =	data_struct.__array__()
								# support for nested types like struct of list of struct
								if isinstance(A	,	np.ndarray					):
															if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
																						return self._consolidate([self.recursive_tensorize(A					) for substruct in data_struct]					)
								elif isinstance(A	,	(list, tuple)					):
															return self._consolidate([self.recursive_tensorize(A					) for substruct in data_struct]					)
								return self._tensorize(A					)





	def 				_lowercase(						self	,	A					)    ->    Dict:
								return map_nested(self._recursive_tensorize	,	A	,	map_list=A					)





	def 				_lowercase(						self	,	A					)    ->    Mapping:
								UpperCAmelCase  :	Union[str, Any]							  =	self.numpy_arrow_extractor().extract_row(A					)
								UpperCAmelCase  :	Dict							  =	self.python_features_decoder.decode_row(A					)
								return self.recursive_tensorize(A					)





	def 				_lowercase(						self	,	A					)    ->    "jax.Array":
								UpperCAmelCase  :	int							  =	self.numpy_arrow_extractor().extract_column(A					)
								UpperCAmelCase  :	Optional[Any]							  =	self.python_features_decoder.decode_column(A	,	pa_table.column_names[0]					)
								UpperCAmelCase  :	Optional[int]							  =	self.recursive_tensorize(A					)
								UpperCAmelCase  :	Any							  =	self._consolidate(A					)
								return column






	def 				_lowercase(						self	,	A					)    ->    Mapping:
								UpperCAmelCase  :	Optional[int]							  =	self.numpy_arrow_extractor().extract_batch(A					)
								UpperCAmelCase  :	List[str]							  =	self.python_features_decoder.decode_batch(A					)
								UpperCAmelCase  :	Union[str, Any]							  =	self.recursive_tensorize(A					)
								for column_name in batch:
															UpperCAmelCase  :	Optional[Any]							  =	self._consolidate(batch[column_name]					)
								return batch

