





import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config

@pytest.fixture(scope='session' )
def       A__(		):
				_snake_case :			int  						=   10
				_snake_case :			Tuple  						=   datasets.Features(
				    {
				        'tokens': datasets.Sequence(datasets.Value('string' ) ),
				        'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive'] ) ),
				        'answers': datasets.Sequence(
				            {
				                'text': datasets.Value('string' ),
				                'answer_start': datasets.Value('int32' ),
				            } ),
				        'id': datasets.Value('int64' ),
				    } )
				_snake_case :			Tuple  						=   datasets.Dataset.from_dict(
				    {
				        'tokens': [['foo'] * 5] * n,
				        'labels': [[1] * 5] * n,
				        'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
				        'id': list(range(a_ ) ),
				    }	,			features=a_	,			)
				return dataset

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Optional[Any]  						=   str(tmp_path_factory.mktemp('data' ) / 'file.arrow' )
				dataset.map(cache_file_name=a_ )
				return filename


# FILE_CONTENT + files


lowercase_		:				Tuple				=						'''\
    Text data.
    Second line of data.'''

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			int  						=   tmp_path_factory.mktemp('data' ) / '''file.txt'''
				_snake_case :			Optional[int]  						=   FILE_CONTENT
				with open(a_	,			'w' ) as f:
								f.write(a_ )
				return filename

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				import bza

				_snake_case :			List[Any]  						=   tmp_path_factory.mktemp('data' ) / '''file.txt.bz2'''
				_snake_case :			Any  						=   bytes(a_	,			'utf-8' )
				with bza.open(a_	,			'wb' ) as f:
								f.write(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				import gzip

				_snake_case :			Any  						=   str(tmp_path_factory.mktemp('data' ) / 'file.txt.gz' )
				_snake_case :			str  						=   bytes(a_	,			'utf-8' )
				with gzip.open(a_	,			'wb' ) as f:
								f.write(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				if datasets.config.LZ4_AVAILABLE:
								import lza.frame

								_snake_case :			Union[str, Any]  						=   tmp_path_factory.mktemp('data' ) / '''file.txt.lz4'''
								_snake_case :			List[Any]  						=   bytes(a_	,			'utf-8' )
								with lza.frame.open(a_	,			'wb' ) as f:
												f.write(a_ )
								return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				if datasets.config.PY7ZR_AVAILABLE:
								import pyazr

								_snake_case :			Dict  						=   tmp_path_factory.mktemp('data' ) / '''file.txt.7z'''
								with pyazr.SevenZipFile(a_	,			'w' ) as archive:
												archive.write(a_	,			arcname=os.path.basename(a_ ) )
								return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				import tarfile

				_snake_case :			Tuple  						=   tmp_path_factory.mktemp('data' ) / '''file.txt.tar'''
				with tarfile.TarFile(a_	,			'w' ) as f:
								f.add(a_	,			arcname=os.path.basename(a_ ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				import lzma

				_snake_case :			int  						=   tmp_path_factory.mktemp('data' ) / '''file.txt.xz'''
				_snake_case :			Tuple  						=   bytes(a_	,			'utf-8' )
				with lzma.open(a_	,			'wb' ) as f:
								f.write(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				import zipfile

				_snake_case :			Dict  						=   tmp_path_factory.mktemp('data' ) / '''file.txt.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.basename(a_ ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				if datasets.config.ZSTANDARD_AVAILABLE:
								import zstandard as zstd

								_snake_case :			str  						=   tmp_path_factory.mktemp('data' ) / '''file.txt.zst'''
								_snake_case :			Tuple  						=   bytes(a_	,			'utf-8' )
								with zstd.open(a_	,			'wb' ) as f:
												f.write(a_ )
								return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			str  						=   tmp_path_factory.mktemp('data' ) / '''file.xml'''
				_snake_case :			int  						=   textwrap.dedent(
				    '\\n    <?xml version="1.0" encoding="UTF-8" ?>\n    <tmx version="1.4">\n      <header segtype="sentence" srclang="ca" />\n      <body>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>' )
				with open(a_	,			'w' ) as f:
								f.write(a_ )
				return filename


lowercase_		:				Tuple				=						[
    {'''col_1''': '''0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''3''', '''col_2''': 3, '''col_3''': 3.0},
]
lowercase_		:				List[str]				=						[
    {'''col_1''': '''4''', '''col_2''': 4, '''col_3''': 4.0},
    {'''col_1''': '''5''', '''col_2''': 5, '''col_3''': 5.0},
]
lowercase_		:				Dict				=						{
    '''col_1''': ['''0''', '''1''', '''2''', '''3'''],
    '''col_2''': [0, 1, 2, 3],
    '''col_3''': [0.0, 1.0, 2.0, 3.0],
}

lowercase_		:				Tuple				=						[
    {'''col_3''': 0.0, '''col_1''': '''0''', '''col_2''': 0},
    {'''col_3''': 1.0, '''col_1''': '''1''', '''col_2''': 1},
]

lowercase_		:				Dict				=						[
    {'''col_1''': '''s0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''s1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''s2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''s3''', '''col_2''': 3, '''col_3''': 3.0},
]

@pytest.fixture(scope='session' )
def       A__(		):
				return DATA_DICT_OF_LISTS

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			List[str]  						=   datasets.Dataset.from_dict(a_ )
				_snake_case :			Tuple  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.arrow' )
				dataset.map(cache_file_name=a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			int  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.sqlite' )
				with contextlib.closing(sqlitea.connect(a_ ) ) as con:
								_snake_case :			Union[str, Any]  						=   con.cursor()
								cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)' )
								for item in DATA:
												cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'	,			tuple(item.values() ) )
								con.commit()
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Any  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.csv' )
				with open(a_	,			'w'	,			newline='' ) as f:
								_snake_case :			Optional[int]  						=   csv.DictWriter(a_	,			fieldnames=['col_1', 'col_2', 'col_3'] )
								writer.writeheader()
								for item in DATA:
												writer.writerow(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Optional[Any]  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset2.csv' )
				with open(a_	,			'w'	,			newline='' ) as f:
								_snake_case :			Dict  						=   csv.DictWriter(a_	,			fieldnames=['col_1', 'col_2', 'col_3'] )
								writer.writeheader()
								for item in DATA:
												writer.writerow(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				import bza

				_snake_case :			Optional[Any]  						=   tmp_path_factory.mktemp('data' ) / '''dataset.csv.bz2'''
				with open(a_	,			'rb' ) as f:
								_snake_case :			Union[str, Any]  						=   f.read()
				# data = bytes(FILE_CONTENT, "utf-8")
				with bza.open(a_	,			'wb' ) as f:
								f.write(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Optional[Any]  						=   tmp_path_factory.mktemp('data' ) / '''dataset.csv.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.basename(a_ ) )
								f.write(a_	,			arcname=os.path.basename(a_ ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Any  						=   tmp_path_factory.mktemp('data' ) / '''dataset.csv.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.basename(csv_path.replace('.csv'	,			'.CSV' ) ) )
								f.write(a_	,			arcname=os.path.basename(csva_path.replace('.csv'	,			'.CSV' ) ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			List[str]  						=   tmp_path_factory.mktemp('data' ) / '''dataset_with_dir.csv.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.join('main_dir'	,			os.path.basename(a_ ) ) )
								f.write(a_	,			arcname=os.path.join('main_dir'	,			os.path.basename(a_ ) ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Tuple  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.parquet' )
				_snake_case :			Any  						=   pa.schema(
				    {
				        'col_1': pa.string(),
				        'col_2': pa.intaa(),
				        'col_3': pa.floataa(),
				    } )
				with open(a_	,			'wb' ) as f:
								_snake_case :			Dict  						=   pq.ParquetWriter(a_	,			schema=a_ )
								_snake_case :			Tuple  						=   pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(a_ ) )] for k in DATA[0]}	,			schema=a_ )
								writer.write_table(a_ )
								writer.close()
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			List[str]  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.json' )
				_snake_case :			int  						=   {'''data''': DATA}
				with open(a_	,			'w' ) as f:
								json.dump(a_	,			a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Optional[int]  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.json' )
				_snake_case :			str  						=   {'''data''': DATA_DICT_OF_LISTS}
				with open(a_	,			'w' ) as f:
								json.dump(a_	,			a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Optional[Any]  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.jsonl' )
				with open(a_	,			'w' ) as f:
								for item in DATA:
												f.write(json.dumps(a_ ) + '\n' )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			int  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset2.jsonl' )
				with open(a_	,			'w' ) as f:
								for item in DATA:
												f.write(json.dumps(a_ ) + '\n' )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Dict  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset_312.jsonl' )
				with open(a_	,			'w' ) as f:
								for item in DATA_312:
												f.write(json.dumps(a_ ) + '\n' )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Dict  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset-str.jsonl' )
				with open(a_	,			'w' ) as f:
								for item in DATA_STR:
												f.write(json.dumps(a_ ) + '\n' )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				import gzip

				_snake_case :			int  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.txt.gz' )
				with open(a_	,			'rb' ) as orig_file:
								with gzip.open(a_	,			'wb' ) as zipped_file:
												zipped_file.writelines(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				import gzip

				_snake_case :			List[Any]  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.jsonl.gz' )
				with open(a_	,			'rb' ) as orig_file:
								with gzip.open(a_	,			'wb' ) as zipped_file:
												zipped_file.writelines(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Dict  						=   tmp_path_factory.mktemp('data' ) / '''dataset.jsonl.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.basename(a_ ) )
								f.write(a_	,			arcname=os.path.basename(a_ ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			List[str]  						=   tmp_path_factory.mktemp('data' ) / '''dataset_nested.jsonl.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.join('nested'	,			os.path.basename(a_ ) ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			List[Any]  						=   tmp_path_factory.mktemp('data' ) / '''dataset_with_dir.jsonl.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.join('main_dir'	,			os.path.basename(a_ ) ) )
								f.write(a_	,			arcname=os.path.join('main_dir'	,			os.path.basename(a_ ) ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			List[Any]  						=   tmp_path_factory.mktemp('data' ) / '''dataset.jsonl.tar'''
				with tarfile.TarFile(a_	,			'w' ) as f:
								f.add(a_	,			arcname=os.path.basename(a_ ) )
								f.add(a_	,			arcname=os.path.basename(a_ ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Union[str, Any]  						=   tmp_path_factory.mktemp('data' ) / '''dataset_nested.jsonl.tar'''
				with tarfile.TarFile(a_	,			'w' ) as f:
								f.add(a_	,			arcname=os.path.join('nested'	,			os.path.basename(a_ ) ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			List[str]  						=   ['''0''', '''1''', '''2''', '''3''']
				_snake_case :			str  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset.txt' )
				with open(a_	,			'w' ) as f:
								for item in data:
												f.write(item + '\n' )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Optional[int]  						=   ['''0''', '''1''', '''2''', '''3''']
				_snake_case :			Any  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset2.txt' )
				with open(a_	,			'w' ) as f:
								for item in data:
												f.write(item + '\n' )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Optional[Any]  						=   ['''0''', '''1''', '''2''', '''3''']
				_snake_case :			Any  						=   tmp_path_factory.mktemp('data' ) / '''dataset.abc'''
				with open(a_	,			'w' ) as f:
								for item in data:
												f.write(item + '\n' )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Dict  						=   tmp_path_factory.mktemp('data' ) / '''dataset.text.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.basename(a_ ) )
								f.write(a_	,			arcname=os.path.basename(a_ ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			int  						=   tmp_path_factory.mktemp('data' ) / '''dataset_with_dir.text.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.join('main_dir'	,			os.path.basename(a_ ) ) )
								f.write(a_	,			arcname=os.path.join('main_dir'	,			os.path.basename(a_ ) ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			int  						=   tmp_path_factory.mktemp('data' ) / '''dataset.ext.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.basename('unsupported.ext' ) )
								f.write(a_	,			arcname=os.path.basename('unsupported_2.ext' ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			Optional[int]  						=   '''\n'''.join(['First', 'Second\u2029with Unicode new line', 'Third'] )
				_snake_case :			int  						=   str(tmp_path_factory.mktemp('data' ) / 'dataset_with_unicode_new_lines.txt' )
				with open(a_	,			'w'	,			encoding='utf-8' ) as f:
								f.write(a_ )
				return path

@pytest.fixture(scope='session' )
def       A__(		):
				return os.path.join('tests'	,			'features'	,			'data'	,			'test_image_rgb.jpg' )

@pytest.fixture(scope='session' )
def       A__(		):
				return os.path.join('tests'	,			'features'	,			'data'	,			'test_audio_44100.wav' )

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase	,			__lowerCAmelCase ):
				_snake_case :			Any  						=   tmp_path_factory.mktemp('data' ) / '''dataset.img.zip'''
				with zipfile.ZipFile(a_	,			'w' ) as f:
								f.write(a_	,			arcname=os.path.basename(a_ ) )
								f.write(a_	,			arcname=os.path.basename(a_ ).replace('.jpg'	,			'2.jpg' ) )
				return path

@pytest.fixture(scope='session' )
def       A__(		__lowerCAmelCase ):
				_snake_case :			List[str]  						=   tmp_path_factory.mktemp('data_dir' )

				(data_dir / "subdir").mkdir()
				with open(data_dir / 'subdir' / 'train.txt'	,			'w' ) as f:
								f.write('foo\n' * 10 )
				with open(data_dir / 'subdir' / 'test.txt'	,			'w' ) as f:
								f.write('bar\n' * 10 )
				# hidden file
				with open(data_dir / 'subdir' / '.test.txt'	,			'w' ) as f:
								f.write('bar\n' * 10 )

				# hidden directory
				(data_dir / ".subdir").mkdir()
				with open(data_dir / '.subdir' / 'train.txt'	,			'w' ) as f:
								f.write('foo\n' * 10 )
				with open(data_dir / '.subdir' / 'test.txt'	,			'w' ) as f:
								f.write('bar\n' * 10 )

				return data_dir













lowercase_		:				Tuple				=						'''
# Installazione di Transformers
! pip install transformers datasets
# Per installare dalla fonte invece dell\'ultima versione rilasciata, commenta il comando sopra e
# rimuovi la modalità commento al comando seguente.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

lowercase_		:				Optional[int]				=						[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
lowercase_		:				str				=						{
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}






