





'''simple docstring'''





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class        _lowerCAmelCase    (			UpperCAmelCase_							,    unittest.TestCase							):


  '''simple docstring'''
  a_							:       Optional[Any]      					=CodeGenTokenizer
  a_							:       Optional[int]      					=CodeGenTokenizerFast
  a_							:       Optional[int]      					=True
  a_							:       Union[str, Any]      					={"""add_prefix_space""": True}
  a_							:       int      					=False







  def UpperCamelCase_ (      self      :					Optional[Any]							):


    '''simple docstring'''





    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    _snake_case   :				Tuple														=	[
        'l',
        'o',
        'w',
        'e',
        'r',
        's',
        't',
        'i',
        'd',
        'n',
        '\u0120',
        '\u0120l',
        '\u0120n',
        '\u0120lo',
        '\u0120low',
        'er',
        '\u0120lowest',
        '\u0120newer',
        '\u0120wider',
        '<unk>',
        '<|endoftext|>',
    ]
    _snake_case   :				Any														=	dict(zip(UpperCamelCase      ,				range(len(UpperCamelCase							)							)							)							)
    _snake_case   :				Dict														=	['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
    _snake_case   :				Optional[int]														=	{'unk_token': '<unk>'}

    _snake_case   :				Union[str, Any]														=	os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['vocab_file']							)
    _snake_case   :				Optional[Any]														=	os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['merges_file']							)
    with open(self.vocab_file      ,				'w'      ,				encoding='utf-8'							) as fp:
      fp.write(json.dumps(UpperCamelCase							) + '\n'							)
    with open(self.merges_file      ,				'w'      ,				encoding='utf-8'							) as fp:
      fp.write('\n'.join(UpperCamelCase							)							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				**UpperCamelCase      :					str							):


    '''simple docstring'''





    kwargs.update(self.special_tokens_map							)
    return CodeGenTokenizer.from_pretrained(self.tmpdirname      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				**UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    kwargs.update(self.special_tokens_map							)
    return CodeGenTokenizerFast.from_pretrained(self.tmpdirname      ,				**UpperCamelCase							)







  def UpperCamelCase_ (      self      :					int      ,				UpperCamelCase      :					str							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	'lower newer'
    _snake_case   :				int														=	'lower newer'
    return input_text, output_text







  def UpperCamelCase_ (      self      :					int							):


    '''simple docstring'''





    _snake_case   :				Any														=	CodeGenTokenizer(self.vocab_file      ,				self.merges_file      ,				**self.special_tokens_map							)
    _snake_case   :				Any														=	'lower newer'
    _snake_case   :				str														=	['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
    _snake_case   :				List[Any]														=	tokenizer.tokenize(UpperCamelCase      ,				add_prefix_space=UpperCamelCase							)
    self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

    _snake_case   :				Any														=	tokens + [tokenizer.unk_token]
    _snake_case   :				Any														=	[14, 15, 10, 9, 3, 2, 15, 19]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase							)      ,				UpperCamelCase							)







  def UpperCamelCase_ (      self      :					int							):


    '''simple docstring'''





    if not self.test_rust_tokenizer:
      return

    _snake_case   :				Any														=	self.get_tokenizer()
    _snake_case   :				Union[str, Any]														=	self.get_rust_tokenizer(add_prefix_space=UpperCamelCase							)

    _snake_case   :				Optional[int]														=	'lower newer'

    # Testing tokenization
    _snake_case   :				Union[str, Any]														=	tokenizer.tokenize(UpperCamelCase      ,				add_prefix_space=UpperCamelCase							)
    _snake_case   :				Union[str, Any]														=	rust_tokenizer.tokenize(UpperCamelCase							)
    self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

    # Testing conversion to ids without special tokens
    _snake_case   :				Optional[Any]														=	tokenizer.encode(UpperCamelCase      ,				add_special_tokens=UpperCamelCase      ,				add_prefix_space=UpperCamelCase							)
    _snake_case   :				str														=	rust_tokenizer.encode(UpperCamelCase      ,				add_special_tokens=UpperCamelCase							)
    self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

    # Testing conversion to ids with special tokens
    _snake_case   :				Tuple														=	self.get_rust_tokenizer(add_prefix_space=UpperCamelCase							)
    _snake_case   :				str														=	tokenizer.encode(UpperCamelCase      ,				add_prefix_space=UpperCamelCase							)
    _snake_case   :				Union[str, Any]														=	rust_tokenizer.encode(UpperCamelCase							)
    self.assertListEqual(UpperCamelCase      ,				UpperCamelCase							)

    # Testing the unknown token
    _snake_case   :				Union[str, Any]														=	tokens + [rust_tokenizer.unk_token]
    _snake_case   :				int														=	[14, 15, 10, 9, 3, 2, 15, 19]
    self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase							)      ,				UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				*UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Optional[int]							):


    '''simple docstring'''





    pass







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					Dict=15							):


    '''simple docstring'''





    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
        _snake_case   :				Union[str, Any]														=	self.rust_tokenizer_class.from_pretrained(UpperCamelCase      ,				**UpperCamelCase							)

        # Simple input
        _snake_case   :				Union[str, Any]														=	'This is a simple input'
        _snake_case   :				Dict														=	['This is a simple input 1', 'This is a simple input 2']
        _snake_case   :				Optional[Any]														=	('This is a simple input', 'This is a pair')
        _snake_case   :				int														=	[
            ('This is a simple input 1', 'This is a simple input 2'),
            ('This is a simple pair 1', 'This is a simple pair 2'),
        ]

        # Simple input tests
        self.assertRaises(UpperCamelCase      ,				tokenizer_r.encode      ,				UpperCamelCase      ,				max_length=UpperCamelCase      ,				padding='max_length'							)

        # Simple input
        self.assertRaises(UpperCamelCase      ,				tokenizer_r.encode_plus      ,				UpperCamelCase      ,				max_length=UpperCamelCase      ,				padding='max_length'							)

        # Simple input
        self.assertRaises(
            UpperCamelCase      ,				tokenizer_r.batch_encode_plus      ,				UpperCamelCase      ,				max_length=UpperCamelCase      ,				padding='max_length'      ,				)

        # Pair input
        self.assertRaises(UpperCamelCase      ,				tokenizer_r.encode      ,				UpperCamelCase      ,				max_length=UpperCamelCase      ,				padding='max_length'							)

        # Pair input
        self.assertRaises(UpperCamelCase      ,				tokenizer_r.encode_plus      ,				UpperCamelCase      ,				max_length=UpperCamelCase      ,				padding='max_length'							)

        # Pair input
        self.assertRaises(
            UpperCamelCase      ,				tokenizer_r.batch_encode_plus      ,				UpperCamelCase      ,				max_length=UpperCamelCase      ,				padding='max_length'      ,				)







  def UpperCamelCase_ (      self      :					Any							):


    '''simple docstring'''





    _snake_case   :				Any														=	CodeGenTokenizer.from_pretrained(self.tmpdirname      ,				pad_token='<pad>'							)

    # Simple input
    _snake_case   :				Optional[int]														=	'This is a simple input'
    _snake_case   :				Optional[Any]														=	['This is a simple input looooooooong', 'This is a simple input']
    _snake_case   :				int														=	('This is a simple input', 'This is a pair')
    _snake_case   :				List[str]														=	[
        ('This is a simple input loooooong', 'This is a simple input'),
        ('This is a simple pair loooooong', 'This is a simple pair'),
    ]

    _snake_case   :				Union[str, Any]														=	tokenizer.pad_token_id

    _snake_case   :				Any														=	tokenizer(UpperCamelCase      ,				padding='max_length'      ,				max_length=30      ,				return_tensors='np'							)
    _snake_case   :				Dict														=	tokenizer(UpperCamelCase      ,				padding=UpperCamelCase      ,				truncate=UpperCamelCase      ,				return_tensors='np'							)
    _snake_case   :				Optional[Any]														=	tokenizer(*UpperCamelCase      ,				padding='max_length'      ,				max_length=60      ,				return_tensors='np'							)
    _snake_case   :				Optional[Any]														=	tokenizer(UpperCamelCase      ,				padding=UpperCamelCase      ,				truncate=UpperCamelCase      ,				return_tensors='np'							)

    # s
    # test single string max_length padding
    self.assertEqual(out_s['input_ids'].shape[-1]      ,				30							)
    self.assertTrue(pad_token_id in out_s['input_ids']							)
    self.assertTrue(0 in out_s['attention_mask']							)

    # s2
    # test automatic padding
    self.assertEqual(out_sa['input_ids'].shape[-1]      ,				33							)
    # long slice doesn't have padding
    self.assertFalse(pad_token_id in out_sa['input_ids'][0]							)
    self.assertFalse(0 in out_sa['attention_mask'][0]							)
    # short slice does have padding
    self.assertTrue(pad_token_id in out_sa['input_ids'][1]							)
    self.assertTrue(0 in out_sa['attention_mask'][1]							)

    # p
    # test single pair max_length padding
    self.assertEqual(out_p['input_ids'].shape[-1]      ,				60							)
    self.assertTrue(pad_token_id in out_p['input_ids']							)
    self.assertTrue(0 in out_p['attention_mask']							)

    # p2
    # test automatic padding pair
    self.assertEqual(out_pa['input_ids'].shape[-1]      ,				52							)
    # long slice pair doesn't have padding
    self.assertFalse(pad_token_id in out_pa['input_ids'][0]							)
    self.assertFalse(0 in out_pa['attention_mask'][0]							)
    # short slice pair does have padding
    self.assertTrue(pad_token_id in out_pa['input_ids'][1]							)
    self.assertTrue(0 in out_pa['attention_mask'][1]							)







  def UpperCamelCase_ (      self      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				str														=	'$$$'
    _snake_case   :				Tuple														=	CodeGenTokenizer.from_pretrained(self.tmpdirname      ,				bos_token=UpperCamelCase      ,				add_bos_token=UpperCamelCase							)

    _snake_case   :				Tuple														=	'This is a simple input'
    _snake_case   :				Tuple														=	['This is a simple input 1', 'This is a simple input 2']

    _snake_case   :				List[Any]														=	tokenizer.bos_token_id

    _snake_case   :				int														=	tokenizer(UpperCamelCase							)
    _snake_case   :				Tuple														=	tokenizer(UpperCamelCase							)

    self.assertEqual(out_s.input_ids[0]      ,				UpperCamelCase							)
    self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

    _snake_case   :				Union[str, Any]														=	tokenizer.decode(out_s.input_ids							)
    _snake_case   :				Any														=	tokenizer.batch_decode(out_sa.input_ids							)

    self.assertEqual(decode_s.split()[0]      ,				UpperCamelCase							)
    self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)







  @slow
  def UpperCamelCase_ (      self      :					str							):


    '''simple docstring'''





    _snake_case   :				Dict														=	CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'							)

    _snake_case   :				Any														=	'\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
    _snake_case   :				Optional[Any]														=	'\nif len_a > len_b:      result = a\nelse:      result = b'

    _snake_case   :				Optional[int]														=	tokenizer.encode(UpperCamelCase							)
    _snake_case   :				Optional[Any]														=	['^#', re.escape('<|endoftext|>'							), '^\'\'\'', '^"""', '\n\n\n']
    _snake_case   :				Optional[Any]														=	tokenizer.decode(UpperCamelCase      ,				truncate_before_pattern=UpperCamelCase							)
    self.assertEqual(UpperCamelCase      ,				UpperCamelCase							)







  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    pass











import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


lowerCAmelCase_						   =    """http://www.mocksite.com/file1.txt"""
lowerCAmelCase_						   =    """\"text\": [\"foo\", \"foo\"]"""
lowerCAmelCase_						   =    """6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8"""



class        _lowerCAmelCase    :


  '''simple docstring'''
  a_							:       int      					=200
  a_							:       List[str]      					={"""Content-Length""": """100"""}
  a_							:       Tuple      					={}







  def UpperCamelCase_ (      self      :					Any      ,				**UpperCamelCase      :					Any							):


    '''simple docstring'''





    return [bytes(UpperCamelCase      ,				'utf-8'							)]




def     lowerCamelCase_	(      *lowerCAmelCase:	Tuple    ,							**lowerCAmelCase:	Tuple				)->		str:
  return MockResponse()


@pytest.mark.parametrize('urls_type'    ,							[str, list, dict]				)
def     lowerCamelCase_	(      lowerCAmelCase:	Union[str, Any]    ,							lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Dict				)->		Optional[Any]:
  import requests

  monkeypatch.setattr(lowerCAmelCase    ,							'request'    ,							lowerCAmelCase				)

  _snake_case   :				List[str]														=	URL
  if issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Optional[int]														=	url
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Any														=	[url]
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Optional[Any]														=	{'train': url}
  _snake_case   :				int														=	'dummy'
  _snake_case   :				Optional[Any]														=	'downloads'
  _snake_case   :				Union[str, Any]														=	tmp_path
  _snake_case   :				Dict														=	DownloadConfig(
      cache_dir=os.path.join(lowerCAmelCase    ,							lowerCAmelCase				)    ,							use_etag=lowerCAmelCase    ,							)
  _snake_case   :				str														=	DownloadManager(dataset_name=lowerCAmelCase    ,							download_config=lowerCAmelCase				)
  _snake_case   :				Optional[int]														=	dl_manager.download(lowerCAmelCase				)
  _snake_case   :				Tuple														=	urls
  for downloaded_paths in [downloaded_paths]:
    if isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      _snake_case   :				Optional[Any]														=	[downloaded_paths]
      _snake_case   :				List[str]														=	[urls]
    elif isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      assert "train" in downloaded_paths.keys()
      _snake_case   :				Any														=	downloaded_paths.values()
      _snake_case   :				List[str]														=	urls.values()
    assert downloaded_paths
    for downloaded_path, input_url in zip(lowerCAmelCase    ,							lowerCAmelCase				):
      assert downloaded_path == dl_manager.downloaded_paths[input_url]
      _snake_case   :				str														=	Path(lowerCAmelCase				)
      _snake_case   :				int														=	downloaded_path.parts
      assert parts[-1] == HASH
      assert parts[-2] == cache_subdir
      assert downloaded_path.exists()
      _snake_case   :				List[str]														=	downloaded_path.read_text()
      assert content == CONTENT
      _snake_case   :				Any														=	downloaded_path.with_suffix('.json'				)
      assert metadata_downloaded_path.exists()
      _snake_case   :				Tuple														=	json.loads(metadata_downloaded_path.read_text()				)
      assert metadata_content == {"url": URL, "etag": None}


@pytest.mark.parametrize('paths_type'    ,							[str, list, dict]				)
def     lowerCamelCase_	(      lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	Optional[int]    ,							lowerCAmelCase:	Any				)->		str:
  _snake_case   :				str														=	str(lowerCAmelCase				)
  if issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				str														=	filename
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				List[Any]														=	[filename]
  elif issubclass(lowerCAmelCase    ,							lowerCAmelCase				):
    _snake_case   :				Optional[Any]														=	{'train': filename}
  _snake_case   :				Any														=	'dummy'
  _snake_case   :				Union[str, Any]														=	xz_file.parent
  _snake_case   :				int														=	'extracted'
  _snake_case   :				Union[str, Any]														=	DownloadConfig(
      cache_dir=lowerCAmelCase    ,							use_etag=lowerCAmelCase    ,							)
  _snake_case   :				List[str]														=	DownloadManager(dataset_name=lowerCAmelCase    ,							download_config=lowerCAmelCase				)
  _snake_case   :				Dict														=	dl_manager.extract(lowerCAmelCase				)
  _snake_case   :				Optional[int]														=	paths
  for extracted_paths in [extracted_paths]:
    if isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      _snake_case   :				List[str]														=	[extracted_paths]
      _snake_case   :				int														=	[paths]
    elif isinstance(lowerCAmelCase    ,							lowerCAmelCase				):
      assert "train" in extracted_paths.keys()
      _snake_case   :				Optional[int]														=	extracted_paths.values()
      _snake_case   :				str														=	paths.values()
    assert extracted_paths
    for extracted_path, input_path in zip(lowerCAmelCase    ,							lowerCAmelCase				):
      assert extracted_path == dl_manager.extracted_paths[input_path]
      _snake_case   :				List[str]														=	Path(lowerCAmelCase				)
      _snake_case   :				Optional[Any]														=	extracted_path.parts
      assert parts[-1] == hash_url_to_filename(lowerCAmelCase    ,							etag=lowerCAmelCase				)
      assert parts[-2] == extracted_subdir
      assert extracted_path.exists()
      _snake_case   :				Optional[int]														=	extracted_path.read_text()
      _snake_case   :				int														=	text_file.read_text()
      assert extracted_file_content == expected_file_content


def     lowerCamelCase_	(      lowerCAmelCase:	Optional[Any]    ,							lowerCAmelCase:	List[Any]				)->		Dict:
  assert path.endswith('.jsonl'				)
  for num_items, line in enumerate(lowerCAmelCase    ,							start=1				):
    _snake_case   :				Dict														=	json.loads(line.decode('utf-8'				)				)
    assert item.keys() == {"col_1", "col_2", "col_3"}
  assert num_items == 4


@pytest.mark.parametrize('archive_jsonl'    ,							['tar_jsonl_path', 'zip_jsonl_path']				)
def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	List[str]				)->		Dict:
  _snake_case   :				List[str]														=	request.getfixturevalue(lowerCAmelCase				)
  _snake_case   :				Optional[Any]														=	DownloadManager()
  for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(lowerCAmelCase				)    ,							start=1				):
    _test_jsonl(lowerCAmelCase    ,							lowerCAmelCase				)
  assert num_jsonl == 2


@pytest.mark.parametrize('archive_nested_jsonl'    ,							['tar_nested_jsonl_path', 'zip_nested_jsonl_path']				)
def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	int				)->		str:
  _snake_case   :				List[Any]														=	request.getfixturevalue(lowerCAmelCase				)
  _snake_case   :				Optional[int]														=	DownloadManager()
  for num_tar, (path, file) in enumerate(dl_manager.iter_archive(lowerCAmelCase				)    ,							start=1				):
    for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(lowerCAmelCase				)    ,							start=1				):
      _test_jsonl(lowerCAmelCase    ,							lowerCAmelCase				)
  assert num_tar == 1
  assert num_jsonl == 2







def     lowerCamelCase_	(      lowerCAmelCase:	Any				)->		int:
  _snake_case   :				Tuple														=	DownloadManager()
  for num_file, file in enumerate(dl_manager.iter_files(lowerCAmelCase				)    ,							start=1				):
    assert os.path.basename(lowerCAmelCase				) == ("test.txt" if num_file == 1 else "train.txt")
  assert num_file == 2




