

import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin







class 	lowercase    (				_lowercase    ,							unittest.TestCase							):


		"""simple docstring"""




		a__					       =  CTRLTokenizer
		a__					       =  False
		a__					       =  False


		def 		A__					(				self):
			super().setUp()

			# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
			_UpperCamelCase   :		Union[str, Any]			=						['adapt', 're@@', 'a@@', 'apt', 'c@@', 't', '<unk>']
			_UpperCamelCase   :		int			=						dict(zip(__snake_case	,       range(len(__snake_case))))
			_UpperCamelCase   :		List[Any]			=						['#version: 0.2', 'a p', 'ap t</w>', 'r e', 'a d', 'ad apt</w>', '']
			_UpperCamelCase   :		int			=						{'unk_token': '<unk>'}

			_UpperCamelCase   :		Tuple			=						os.path.join(self.tmpdirname	,       VOCAB_FILES_NAMES['vocab_file'])
			_UpperCamelCase   :		List[Any]			=						os.path.join(self.tmpdirname	,       VOCAB_FILES_NAMES['merges_file'])
			with open(self.vocab_file	,       'w'	,       encoding='utf-8') as fp:
				fp.write(json.dumps(__snake_case) + '\n')
			with open(self.merges_file	,       'w'	,       encoding='utf-8') as fp:
				fp.write('\n'.join(__snake_case))


		def 		A__					(				self	,       **__snake_case):
			kwargs.update(self.special_tokens_map)
			return CTRLTokenizer.from_pretrained(self.tmpdirname	,       **__snake_case)


		def 		A__					(				self	,       __snake_case):
			_UpperCamelCase   :		Union[str, Any]			=						'adapt react readapt apt'
			_UpperCamelCase   :		Optional[int]			=						'adapt react readapt apt'
			return input_text, output_text






		def 		A__					(				self):
			_UpperCamelCase   :		List[Any]			=						CTRLTokenizer(self.vocab_file	,       self.merges_file	,       **self.special_tokens_map)
			_UpperCamelCase   :		Optional[Any]			=						'adapt react readapt apt'
			_UpperCamelCase   :		Dict			=						'adapt re@@ a@@ c@@ t re@@ adapt apt'.split()
			_UpperCamelCase   :		List[str]			=						tokenizer.tokenize(__snake_case)
			self.assertListEqual(__snake_case	,       __snake_case)

			_UpperCamelCase   :		int			=						tokens + [tokenizer.unk_token]

			_UpperCamelCase   :		Union[str, Any]			=						[0, 1, 2, 4, 5, 1, 0, 3, 6]
			self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case)	,       __snake_case)







# tests directory-specific settings - this file is run automatically
# by pytest before any tests are run

import doctest
import sys
import warnings
from os.path import abspath, dirname, join

import _pytest

from transformers.testing_utils import HfDoctestModule, HfDocTestParser


# allow having multiple repository checkouts and not needing to remember to rerun
# 'pip install -e .[dev]' when switching between checkouts and running tests.
lowerCAmelCase__								=		abspath(join(dirname(__file__), """src"""))
sys.path.insert(1, git_repo_path)

# silence FutureWarning warnings in tests since often we can't act on them until
# they become normal warnings - i.e. the tests still need to test the current functionality
warnings.simplefilter(action="""ignore""", category=FutureWarning)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Any	)						->				Union[str, Any]:




	'''simple docstring'''




	config.addinivalue_line(
	    'markers'							,       'is_pt_tf_cross_test: mark test to run only when PT and TF interactions are tested'	)
	config.addinivalue_line(
	    'markers'							,       'is_pt_flax_cross_test: mark test to run only when PT and FLAX interactions are tested'	)
	config.addinivalue_line('markers'							,       'is_pipeline_test: mark test to run only when pipelines are tested'	)
	config.addinivalue_line('markers'							,       'is_staging_test: mark test to run only in the staging environment'	)
	config.addinivalue_line('markers'							,       'accelerate_tests: mark test that require accelerate'	)
	config.addinivalue_line('markers'							,       'tool_tests: mark the tool tests that are run on their specific schedule'	)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Optional[int]	)						->				List[Any]:




	'''simple docstring'''




	from transformers.testing_utils import pytest_addoption_shared

	pytest_addoption_shared(UpperCAmelCase_	)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Any	)						->				List[Any]:




	'''simple docstring'''




	from transformers.testing_utils import pytest_terminal_summary_main

	_UpperCamelCase   :		str			=						terminalreporter.config.getoption('--make-reports'	)
	if make_reports:
		pytest_terminal_summary_main(UpperCAmelCase_							,       id=UpperCAmelCase_	)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Dict							,       UpperCAmelCase_							:     Dict	)						->				Tuple:




	'''simple docstring'''




	if exitstatus == 5:
		_UpperCamelCase   :		List[Any]			=						0


# Doctest custom flag to ignore output.
lowerCAmelCase__								=		doctest.register_optionflag("""IGNORE_RESULT""")

lowerCAmelCase__								=		doctest.OutputChecker







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		def 		A__					(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			if IGNORE_RESULT & optionflags:
				return True
			return OutputChecker.check_output(self	,       __snake_case	,       __snake_case	,       __snake_case)

lowerCAmelCase__								=		CustomOutputChecker
lowerCAmelCase__								=		HfDoctestModule







lowerCAmelCase__								=		HfDocTestParser




