






"""simple docstring"""




UpperCamelCase							       =    'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'






def        lowerCAmelCase (      UpperCamelCase_:							bytes					)       ->							bytes:







		'''simple docstring'''







		if not isinstance(_A     , _A					):
				_a							=	f'''a bytes-like object is required, not \'{data.__class__.__name__}\''''
				raise TypeError(_A					)

		_a							=	"".join(bin(_A					)[2:].zfill(8					) for byte in data					)

		_a							=	len(_A					) % 6 != 0

		if padding_needed:
				# The padding that will be added later
				_a							=	b"=" * ((6 - len(_A					) % 6) // 2)

				# Append binary_stream with arbitrary binary digits (0's by default) to make its
				# length a multiple of 6.
				binary_stream += "0" * (6 - len(_A					) % 6)
		else:
				_a							=	b""

		# Encode every 6 binary digits to their corresponding Base64 character
		return (
		    "".join(
		        B64_CHARSET[int(binary_stream[index : index + 6]     , 2					)]
		        for index in range(0     , len(_A					)     , 6					)					).encode()
		    + padding
		)






def        lowerCAmelCase (      UpperCamelCase_:							str					)       ->							bytes:







		'''simple docstring'''







		if not isinstance(_A     , _A					) and not isinstance(_A     , _A					):
				_a							=	(
				    "argument should be a bytes-like object or ASCII string, "
				    f'''not \'{encoded_data.__class__.__name__}\''''
				)
				raise TypeError(_A					)

		# In case encoded_data is a bytes-like object, make sure it contains only
		# ASCII characters so we convert it to a string object
		if isinstance(_A     , _A					):
				try:
						_a							=	encoded_data.decode("utf-8"					)
				except UnicodeDecodeError:
						raise ValueError("base64 encoded data should only contain ASCII characters"					)

		_a							=	encoded_data.count("="					)

		# Check if the encoded string contains non base64 characters
		if padding:
				assert all(
				    char in B64_CHARSET for char in encoded_data[:-padding]					), "Invalid base64 character(s) found."
		else:
				assert all(
				    char in B64_CHARSET for char in encoded_data					), "Invalid base64 character(s) found."

		# Check the padding
		assert len(_A					) % 4 == 0 and padding < 3, "Incorrect padding"

		if padding:
				# Remove padding if there is one
				_a							=	encoded_data[:-padding]

				_a							=	"".join(
				    bin(B64_CHARSET.index(_A					)					)[2:].zfill(6					) for char in encoded_data					)[: -padding * 2]
		else:
				_a							=	"".join(
				    bin(B64_CHARSET.index(_A					)					)[2:].zfill(6					) for char in encoded_data					)

		_a							=	[
		    int(binary_stream[index : index + 8]     , 2					)
		    for index in range(0     , len(_A					)     , 8					)
		]

		return bytes(_A					)


if __name__ == "__main__":
					import doctest

					doctest.testmod()










"""simple docstring"""




import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






def        lowerCAmelCase (      UpperCamelCase_:							Features					)       ->							Optional[int]:







  '''simple docstring'''







  _a							=	np.inf

  def set_batch_size(UpperCamelCase_:							FeatureType					) -> None:
    nonlocal batch_size
    if isinstance(UpperCamelCase_     , UpperCamelCase_					):
      _a							=	min(UpperCamelCase_     , config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS					)
    elif isinstance(UpperCamelCase_     , UpperCamelCase_					):
      _a							=	min(UpperCamelCase_     , config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS					)
    elif isinstance(UpperCamelCase_     , UpperCamelCase_					) and feature.dtype == "binary":
      _a							=	min(UpperCamelCase_     , config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS					)

  _visit(UpperCamelCase_     , UpperCamelCase_					)

  return None if batch_size is np.inf else batch_size




class    lowercase_    (_UpperCAmelCase     ):






      def __init__(    self     ,     a_     ,     a_ = None     ,     a_ = None     ,     a_ = None     ,     a_ = False     ,     a_ = False     ,     a_ = None     ,     **a_     ,     )	->Optional[int]:







        '''simple docstring'''


        super().__init__(
            a_     ,     split=a_     ,     features=a_     ,     cache_dir=a_     ,     keep_in_memory=a_     ,     streaming=a_     ,     num_proc=a_     ,     **a_     ,     )
        _a							=	path_or_paths if isinstance(a_     ,     a_	) else {self.split: path_or_paths}
        _a							=	_PACKAGED_DATASETS_MODULES["parquet"][1]
        _a							=	Parquet(
            cache_dir=a_     ,     data_files=a_     ,     features=a_     ,     hash=a_     ,     **a_     ,     )






      def 			lowerCamelCase__    (    self	)	->str:







        '''simple docstring'''


        if self.streaming:
          _a							=	self.builder.as_streaming_dataset(split=self.split	)
        # Build regular (map-style) dataset
        else:
          _a							=	None
          _a							=	None
          _a							=	None
          _a							=	None

          self.builder.download_and_prepare(
              download_config=a_     ,     download_mode=a_     ,     verification_mode=a_     ,     base_path=a_     ,     num_proc=self.num_proc     ,     )
          _a							=	self.builder.as_dataset(
              split=self.split     ,     verification_mode=a_     ,     in_memory=self.keep_in_memory	)
        return dataset







class    lowercase_    :






      def __init__(    self     ,     a_     ,     a_     ,     a_ = None     ,     **a_     ,     )	->int:







        '''simple docstring'''


        _a							=	dataset
        _a							=	path_or_buf
        _a							=	batch_size or get_writer_batch_size(dataset.features	)
        _a							=	parquet_writer_kwargs






      def 			lowerCamelCase__    (    self	)	->int:







        '''simple docstring'''


        _a							=	self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

        if isinstance(self.path_or_buf     ,     (str, bytes, os.PathLike)	):
          with open(self.path_or_buf     ,     "wb+"	) as buffer:
            _a							=	self._write(file_obj=a_     ,     batch_size=a_     ,     **self.parquet_writer_kwargs	)
        else:
          _a							=	self._write(file_obj=self.path_or_buf     ,     batch_size=a_     ,     **self.parquet_writer_kwargs	)
        return written






      def 			lowerCamelCase__    (    self     ,     a_     ,     a_     ,     **a_	)	->int:







        '''simple docstring'''


        _a							=	0
        _a							=	parquet_writer_kwargs.pop("path_or_buf"     ,     a_	)
        _a							=	self.dataset.features.arrow_schema

        _a							=	pq.ParquetWriter(a_     ,     schema=a_     ,     **a_	)

        for offset in logging.tqdm(
            range(0     ,     len(self.dataset	)     ,     a_	)     ,     unit="ba"     ,     disable=not logging.is_progress_bar_enabled()     ,     desc="Creating parquet from Arrow format"     ,     ):
          _a							=	query_table(
              table=self.dataset._data     ,     key=slice(a_     ,     offset + batch_size	)     ,     indices=self.dataset._indices if self.dataset._indices is not None else None     ,     )
          writer.write_table(a_	)
          written += batch.nbytes
        writer.close()
        return written


