





import collections
import inspect
import unittest

from transformers import FocalNetConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch
       from torch import nn

       from transformers import (
           FocalNetBackbone,
           FocalNetForImageClassification,
           FocalNetForMaskedImageModeling,
           FocalNetModel,
       )
       from transformers.models.focalnet.modeling_focalnet import FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
       from PIL import Image

       from transformers import AutoImageProcessor







class 			__A       :




      """simple docstring"""







      def __init__(		self      ,							a__      ,							a__=13      ,							a__=32      ,							a__=2      ,							a__=3      ,							a__=16      ,							a__=[32, 64, 128]      ,							a__=[1, 2, 1]      ,							a__=[2, 2, 4]      ,							a__=2      ,							a__=2.0      ,							a__=True      ,							a__=0.0      ,							a__=0.0      ,							a__=0.1      ,							a__="gelu"      ,							a__=False      ,							a__=True      ,							a__=0.02      ,							a__=1e-5      ,							a__=True      ,							a__=None      ,							a__=True      ,							a__=10      ,							a__=8      ,							a__=["stage1", "stage2"]      ,							a__=[1, 2]      ,							):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		parent
             _lowerCamelCase							:     Union[str, Any]	=		batch_size
             _lowerCamelCase							:     Dict	=		image_size
             _lowerCamelCase							:     Dict	=		patch_size
             _lowerCamelCase							:     Any	=		num_channels
             _lowerCamelCase							:     Optional[int]	=		embed_dim
             _lowerCamelCase							:     List[str]	=		hidden_sizes
             _lowerCamelCase							:     str	=		depths
             _lowerCamelCase							:     Dict	=		num_heads
             _lowerCamelCase							:     str	=		window_size
             _lowerCamelCase							:     Tuple	=		mlp_ratio
             _lowerCamelCase							:     Optional[Any]	=		qkv_bias
             _lowerCamelCase							:     List[str]	=		hidden_dropout_prob
             _lowerCamelCase							:     int	=		attention_probs_dropout_prob
             _lowerCamelCase							:     Dict	=		drop_path_rate
             _lowerCamelCase							:     int	=		hidden_act
             _lowerCamelCase							:     Optional[Any]	=		use_absolute_embeddings
             _lowerCamelCase							:     Union[str, Any]	=		patch_norm
             _lowerCamelCase							:     Union[str, Any]	=		layer_norm_eps
             _lowerCamelCase							:     Optional[Any]	=		initializer_range
             _lowerCamelCase							:     List[Any]	=		is_training
             _lowerCamelCase							:     Optional[int]	=		scope
             _lowerCamelCase							:     Optional[int]	=		use_labels
             _lowerCamelCase							:     str	=		type_sequence_label_size
             _lowerCamelCase							:     Optional[int]	=		encoder_stride
             _lowerCamelCase							:     List[str]	=		out_features
             _lowerCamelCase							:     Dict	=		out_indices


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

             _lowerCamelCase							:     str	=		None
             if self.use_labels:
                    _lowerCamelCase							:     Optional[Any]	=		ids_tensor([self.batch_size]      ,							self.type_sequence_label_size)

             _lowerCamelCase							:     Optional[Any]	=		self.get_config()

             return config, pixel_values, labels


      def 				__snake_case			(		self):



             """simple docstring"""





             return FocalNetConfig(
                 image_size=self.image_size      ,							patch_size=self.patch_size      ,							num_channels=self.num_channels      ,							embed_dim=self.embed_dim      ,							hidden_sizes=self.hidden_sizes      ,							depths=self.depths      ,							num_heads=self.num_heads      ,							window_size=self.window_size      ,							mlp_ratio=self.mlp_ratio      ,							qkv_bias=self.qkv_bias      ,							hidden_dropout_prob=self.hidden_dropout_prob      ,							attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,							drop_path_rate=self.drop_path_rate      ,							hidden_act=self.hidden_act      ,							use_absolute_embeddings=self.use_absolute_embeddings      ,							path_norm=self.patch_norm      ,							layer_norm_eps=self.layer_norm_eps      ,							initializer_range=self.initializer_range      ,							encoder_stride=self.encoder_stride      ,							out_features=self.out_features      ,							out_indices=self.out_indices      ,							)


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		FocalNetModel(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     str	=		model(a__)

             _lowerCamelCase							:     Union[str, Any]	=		((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths) - 1))
             _lowerCamelCase							:     Dict	=		int(config.embed_dim * 2 ** (len(config.depths) - 1))

             self.parent.assertEqual(result.last_hidden_state.shape      ,							(self.batch_size, expected_seq_len, expected_dim))


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		FocalNetBackbone(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Dict	=		model(a__)

             # verify feature maps
             self.parent.assertEqual(len(result.feature_maps)      ,							len(config.out_features))
             self.parent.assertListEqual(list(result.feature_maps[0].shape)      ,							[self.batch_size, self.image_size, 8, 8])

             # verify channels
             self.parent.assertEqual(len(model.channels)      ,							len(config.out_features))
             self.parent.assertListEqual(model.channels      ,							config.hidden_sizes[:-1])

             # verify backbone works with out_features=None
             _lowerCamelCase							:     str	=		None
             _lowerCamelCase							:     List[str]	=		FocalNetBackbone(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     str	=		model(a__)

             # verify feature maps
             self.parent.assertEqual(len(result.feature_maps)      ,							1)
             self.parent.assertListEqual(list(result.feature_maps[0].shape)      ,							[self.batch_size, self.image_size * 2, 4, 4])

             # verify channels
             self.parent.assertEqual(len(model.channels)      ,							1)
             self.parent.assertListEqual(model.channels      ,							[config.hidden_sizes[-1]])


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		FocalNetForMaskedImageModeling(config=a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Dict	=		model(a__)
             self.parent.assertEqual(
                 result.reconstruction.shape      ,							(self.batch_size, self.num_channels, self.image_size, self.image_size))

             # test greyscale images
             _lowerCamelCase							:     Any	=		1
             _lowerCamelCase							:     Any	=		FocalNetForMaskedImageModeling(a__)
             model.to(a__)
             model.eval()

             _lowerCamelCase							:     List[str]	=		floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
             _lowerCamelCase							:     List[Any]	=		model(a__)
             self.parent.assertEqual(result.reconstruction.shape      ,							(self.batch_size, 1, self.image_size, self.image_size))


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		self.type_sequence_label_size
             _lowerCamelCase							:     List[Any]	=		FocalNetForImageClassification(a__)
             model.to(a__)
             model.eval()
             _lowerCamelCase							:     Optional[Any]	=		model(a__      ,							labels=a__)
             self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.type_sequence_label_size))

             # test greyscale images
             _lowerCamelCase							:     Tuple	=		1
             _lowerCamelCase							:     List[Any]	=		FocalNetForImageClassification(a__)
             model.to(a__)
             model.eval()

             _lowerCamelCase							:     Dict	=		floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
             _lowerCamelCase							:     Any	=		model(a__)
             self.parent.assertEqual(result.logits.shape      ,							(self.batch_size, self.type_sequence_label_size))



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		self.prepare_config_and_inputs()

             _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase							:     Tuple	=		config_and_inputs
             _lowerCamelCase							:     int	=		{'''pixel_values''': pixel_values}
             return config, inputs_dict







@require_torch
class 			__A       (      lowerCamelCase__ ,lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      (
          (
              FocalNetModel,
              FocalNetForImageClassification,
              FocalNetForMaskedImageModeling,
              FocalNetBackbone,
          )
          if is_torch_available()
          else ()
      )
      UpperCAmelCase__						=      (
          {'feature-extraction': FocalNetModel, 'image-classification': FocalNetForImageClassification}
          if is_torch_available()
          else {}
      )
      UpperCAmelCase__						=      False

      UpperCAmelCase__						=      False
      UpperCAmelCase__						=      False
      UpperCAmelCase__						=      False
      UpperCAmelCase__						=      False


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		FocalNetModelTester(self)
             _lowerCamelCase							:     List[str]	=		ConfigTester(self      ,							config_class=a__      ,							embed_dim=37      ,							has_text_modality=a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             self.create_and_test_config_common_properties()
             self.config_tester.create_and_test_config_to_json_string()
             self.config_tester.create_and_test_config_to_json_file()
             self.config_tester.create_and_test_config_from_and_save_pretrained()
             self.config_tester.create_and_test_config_with_num_labels()
             self.config_tester.check_config_can_be_init_without_params()
             self.config_tester.check_config_arguments_init()


      def 				__snake_case			(		self):



             """simple docstring"""





             return


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_backbone(*a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_masked_image_modeling(*a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_image_classification(*a__)


      @unittest.skip(reason='''FocalNet does not use inputs_embeds''')
      def 				__snake_case			(		self):



             """simple docstring"""





             pass


      @unittest.skip(reason='''FocalNet does not use feedforward chunking''')
      def 				__snake_case			(		self):



             """simple docstring"""





             pass


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase,  _lowerCamelCase							:     List[Any]	=		self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes[:-1]:
                    _lowerCamelCase							:     Optional[Any]	=		model_class(a__)
                    self.assertIsInstance(model.get_input_embeddings()      ,							(nn.Module))
                    _lowerCamelCase							:     Any	=		model.get_output_embeddings()
                    self.assertTrue(x is None or isinstance(a__      ,							nn.Linear))


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase,  _lowerCamelCase							:     int	=		self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes[:-1]:
                    _lowerCamelCase							:     Tuple	=		model_class(a__)
                    _lowerCamelCase							:     str	=		inspect.signature(model.forward)
                    # signature.parameters is an OrderedDict => so arg_names order is deterministic
                    _lowerCamelCase							:     Optional[Any]	=		[*signature.parameters.keys()]

                    _lowerCamelCase							:     Dict	=		['''pixel_values''']
                    self.assertListEqual(arg_names[:1]      ,							a__)


      def 				__snake_case			(		self      ,							a__      ,							a__      ,							a__      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		model_class(a__)
             model.to(a__)
             model.eval()

             with torch.no_grad():
                    _lowerCamelCase							:     Any	=		model(**self._prepare_for_class(a__      ,							a__))

             _lowerCamelCase							:     Any	=		outputs.hidden_states

             _lowerCamelCase							:     Optional[int]	=		getattr(
                 self.model_tester      ,							'''expected_num_hidden_layers'''      ,							len(self.model_tester.depths) + 1)
             self.assertEqual(len(a__)      ,							a__)

             # FocalNet has a different seq_length
             _lowerCamelCase							:     Optional[Any]	=		(
                 config.patch_size
                 if isinstance(config.patch_size      ,							collections.abc.Iterable)
                 else (config.patch_size, config.patch_size)
             )

             _lowerCamelCase							:     Union[str, Any]	=		(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

             self.assertListEqual(
                 list(hidden_states[0].shape[-2:])      ,							[num_patches, self.model_tester.embed_dim]      ,							)

             _lowerCamelCase							:     Any	=		outputs.reshaped_hidden_states
             self.assertEqual(len(a__)      ,							a__)

             _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase							:     Tuple	=		reshaped_hidden_states[0].shape
             _lowerCamelCase							:     Optional[Any]	=		(
                 reshaped_hidden_states[0].view(a__      ,							a__      ,							height * width).permute(0      ,							2      ,							1)
             )
             self.assertListEqual(
                 list(reshaped_hidden_states.shape[-2:])      ,							[num_patches, self.model_tester.embed_dim]      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase,  _lowerCamelCase							:     Dict	=		self.model_tester.prepare_config_and_inputs_for_common()

             _lowerCamelCase							:     Any	=		(
                 self.model_tester.image_size
                 if isinstance(self.model_tester.image_size      ,							collections.abc.Iterable)
                 else (self.model_tester.image_size, self.model_tester.image_size)
             )

             for model_class in self.all_model_classes[:-1]:
                    _lowerCamelCase							:     Any	=		True
                    self.check_hidden_states_output(a__      ,							a__      ,							a__      ,							a__)

                    # check that output_hidden_states also work using config
                    del inputs_dict["output_hidden_states"]
                    _lowerCamelCase							:     Optional[Any]	=		True

                    self.check_hidden_states_output(a__      ,							a__      ,							a__      ,							a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase,  _lowerCamelCase							:     Optional[int]	=		self.model_tester.prepare_config_and_inputs_for_common()
             _lowerCamelCase							:     List[str]	=		3

             _lowerCamelCase							:     Union[str, Any]	=		(
                 self.model_tester.image_size
                 if isinstance(self.model_tester.image_size      ,							collections.abc.Iterable)
                 else (self.model_tester.image_size, self.model_tester.image_size)
             )
             _lowerCamelCase							:     str	=		(
                 config.patch_size
                 if isinstance(config.patch_size      ,							collections.abc.Iterable)
                 else (config.patch_size, config.patch_size)
             )

             _lowerCamelCase							:     str	=		image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
             _lowerCamelCase							:     Optional[int]	=		image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

             for model_class in self.all_model_classes[:-1]:
                    _lowerCamelCase							:     Optional[int]	=		True
                    self.check_hidden_states_output(a__      ,							a__      ,							a__      ,							(padded_height, padded_width))

                    # check that output_hidden_states also work using config
                    del inputs_dict["output_hidden_states"]
                    _lowerCamelCase							:     List[str]	=		True
                    self.check_hidden_states_output(a__      ,							a__      ,							a__      ,							(padded_height, padded_width))


      @slow
      def 				__snake_case			(		self):



             """simple docstring"""





             for model_name in FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                    _lowerCamelCase							:     Dict	=		FocalNetModel.from_pretrained(a__)
                    self.assertIsNotNone(a__)



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase,  _lowerCamelCase							:     Union[str, Any]	=		self.model_tester.prepare_config_and_inputs_for_common()

             _lowerCamelCase							:     List[Any]	=		_config_zero_init(a__)
             for model_class in self.all_model_classes:
                    _lowerCamelCase							:     Tuple	=		model_class(config=a__)
                    for name, param in model.named_parameters():
                           if "embeddings" not in name and param.requires_grad:
                                  self.assertIn(
                                      ((param.data.mean() * 1e9).round() / 1e9).item()      ,							[0.0, 1.0]      ,							msg=F"""Parameter {name} of model {model_class} seems not properly initialized"""      ,							)







@require_vision
@require_torch
class 			__A       (      unittest.TestCase  ):




      """simple docstring"""







      @cached_property
      def 				__snake_case			(		self):



             """simple docstring"""





             return AutoImageProcessor.from_pretrained('''microsoft/focalnet-tiny''') if is_vision_available() else None



      @slow
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		FocalNetForImageClassification.from_pretrained('''microsoft/focalnet-tiny''').to(a__)
             _lowerCamelCase							:     List[Any]	=		self.default_image_processor

             _lowerCamelCase							:     Dict	=		Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
             _lowerCamelCase							:     Optional[Any]	=		image_processor(images=a__      ,							return_tensors='''pt''').to(a__)

             # forward pass
             with torch.no_grad():
                    _lowerCamelCase							:     List[Any]	=		model(**a__)

             # verify the logits
             _lowerCamelCase							:     Tuple	=		torch.Size((1, 1000))
             self.assertEqual(outputs.logits.shape      ,							a__)
             _lowerCamelCase							:     Optional[Any]	=		torch.tensor([0.2166, -0.4368, 0.2191]).to(a__)
             self.assertTrue(torch.allclose(outputs.logits[0, :3]      ,							a__      ,							atol=1e-4))
             self.assertTrue(outputs.logits.argmax(dim=-1).item()      ,							281)







@require_torch
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      (FocalNetBackbone,) if is_torch_available() else ()
      UpperCAmelCase__						=      FocalNetConfig

      UpperCAmelCase__						=      False


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		FocalNetModelTester(self)








import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      BertJapaneseTokenizer
      UpperCAmelCase__						=      False
      UpperCAmelCase__						=      True


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             _lowerCamelCase							:     Optional[int]	=		[
                 '''[UNK]''',
                 '''[CLS]''',
                 '''[SEP]''',
                 '''こんにちは''',
                 '''こん''',
                 '''にちは''',
                 '''ばんは''',
                 '''##こん''',
                 '''##にちは''',
                 '''##ばんは''',
                 '''世界''',
                 '''##世界''',
                 '''、''',
                 '''##、''',
                 '''。''',
                 '''##。''',
             ]

             _lowerCamelCase							:     Dict	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''vocab_file'''])
             with open(self.vocab_file      ,							'''w'''      ,							encoding='''utf-8''') as vocab_writer:
                    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		'''こんにちは、世界。 \nこんばんは、世界。'''
             _lowerCamelCase							:     Union[str, Any]	=		'''こんにちは 、 世界 。 こんばんは 、 世界 。'''
             return input_text, output_text


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase,  _lowerCamelCase							:     int	=		self.get_input_output_texts(a__)
             _lowerCamelCase							:     Optional[Any]	=		tokenizer.encode(a__      ,							add_special_tokens=a__)
             _lowerCamelCase							:     str	=		tokenizer.decode(a__      ,							clean_up_tokenization_spaces=a__)
             return text, ids


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		self.tokenizer_class(self.vocab_file)

             _lowerCamelCase							:     Tuple	=		tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。''')
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		self.tokenizer_class(self.vocab_file      ,							word_tokenizer_type='''mecab''')
             self.assertIsNotNone(a__)

             _lowerCamelCase							:     List[str]	=		'''こんにちは、世界。\nこんばんは、世界。'''
             _lowerCamelCase							:     int	=		tokenizer.tokenize(a__)
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

             _lowerCamelCase							:     Optional[int]	=		os.path.join(self.tmpdirname      ,							'''tokenizer.bin''')
             with open(a__      ,							'''wb''') as handle:
                    pickle.dump(a__      ,							a__)

             with open(a__      ,							'''rb''') as handle:
                    _lowerCamelCase							:     List[str]	=		pickle.load(a__)

             _lowerCamelCase							:     Dict	=		tokenizer_new.tokenize(a__)

             self.assertListEqual(a__      ,							a__)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		MecabTokenizer(mecab_dic='''ipadic''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             try:
                    _lowerCamelCase							:     str	=		MecabTokenizer(mecab_dic='''unidic_lite''')
             except ModuleNotFoundError:
                    return

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             try:
                    _lowerCamelCase							:     int	=		MecabTokenizer(mecab_dic='''unidic''')
             except ModuleNotFoundError:
                    return

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		MecabTokenizer(do_lower_case=a__      ,							mecab_dic='''ipadic''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             try:
                    _lowerCamelCase							:     Any	=		MecabTokenizer(
                        do_lower_case=a__      ,							normalize_text=a__      ,							mecab_option='''-d /usr/local/lib/mecab/dic/jumandic''')
             except RuntimeError:
                    # if dict doesn't exist in the system, previous code raises this error.
                    return

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		MecabTokenizer(normalize_text=a__      ,							mecab_dic='''ipadic''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		self.tokenizer_class(self.vocab_file      ,							word_tokenizer_type='''sudachi''')
             self.assertIsNotNone(a__)

             _lowerCamelCase							:     Tuple	=		'''こんにちは、世界。\nこんばんは、世界。'''
             _lowerCamelCase							:     Tuple	=		tokenizer.tokenize(a__)
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

             _lowerCamelCase							:     Tuple	=		os.path.join(self.tmpdirname      ,							'''tokenizer.bin''')
             with open(a__      ,							'''wb''') as handle:
                    pickle.dump(a__      ,							a__)

             with open(a__      ,							'''rb''') as handle:
                    _lowerCamelCase							:     str	=		pickle.load(a__)

             _lowerCamelCase							:     Any	=		tokenizer_new.tokenize(a__)

             self.assertListEqual(a__      ,							a__)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[int]	=		SudachiTokenizer(sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		SudachiTokenizer(sudachi_dict_type='''core'''      ,							sudachi_split_mode='''A''')

             self.assertListEqual(tokenizer.tokenize('''外国人参政権''')      ,							['''外国''', '''人''', '''参政''', '''権'''])


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		SudachiTokenizer(sudachi_dict_type='''core'''      ,							sudachi_split_mode='''B''')

             self.assertListEqual(tokenizer.tokenize('''外国人参政権''')      ,							['''外国人''', '''参政権'''])


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		SudachiTokenizer(sudachi_dict_type='''core'''      ,							sudachi_split_mode='''C''')

             self.assertListEqual(tokenizer.tokenize('''外国人参政権''')      ,							['''外国人参政権'''])


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		SudachiTokenizer(do_lower_case=a__      ,							sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		SudachiTokenizer(normalize_text=a__      ,							sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							[''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']      ,							)


      @require_sudachi
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		SudachiTokenizer(trim_whitespace=a__      ,							sudachi_dict_type='''core''')

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		self.tokenizer_class(self.vocab_file      ,							word_tokenizer_type='''jumanpp''')
             self.assertIsNotNone(a__)

             _lowerCamelCase							:     List[str]	=		'''こんにちは、世界。\nこんばんは、世界。'''
             _lowerCamelCase							:     List[str]	=		tokenizer.tokenize(a__)
             self.assertListEqual(a__      ,							['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。'''])
             self.assertListEqual(tokenizer.convert_tokens_to_ids(a__)      ,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

             _lowerCamelCase							:     List[str]	=		os.path.join(self.tmpdirname      ,							'''tokenizer.bin''')
             with open(a__      ,							'''wb''') as handle:
                    pickle.dump(a__      ,							a__)

             with open(a__      ,							'''rb''') as handle:
                    _lowerCamelCase							:     Optional[Any]	=		pickle.load(a__)

             _lowerCamelCase							:     Dict	=		tokenizer_new.tokenize(a__)

             self.assertListEqual(a__      ,							a__)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		JumanppTokenizer()

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		JumanppTokenizer(do_lower_case=a__)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		JumanppTokenizer(normalize_text=a__)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		JumanppTokenizer(trim_whitespace=a__)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  ''')      ,							['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']      ,							)


      @require_jumanpp
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		JumanppTokenizer()

             self.assertListEqual(
                 tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。''')      ,							['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     str	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

             _lowerCamelCase							:     List[str]	=		{}
             for i, token in enumerate(a__):
                    _lowerCamelCase							:     Optional[Any]	=		i
             _lowerCamelCase							:     Union[str, Any]	=		WordpieceTokenizer(vocab=a__      ,							unk_token='''[UNK]''')

             self.assertListEqual(tokenizer.tokenize('''''')      ,							[])

             self.assertListEqual(tokenizer.tokenize('''こんにちは''')      ,							['''こんにちは'''])

             self.assertListEqual(tokenizer.tokenize('''こんばんは''')      ,							['''こん''', '''##ばんは'''])

             self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは''')      ,							['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは'''])


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Tuple	=		BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp''')
             _lowerCamelCase							:     str	=		tokenizer.subword_tokenizer

             _lowerCamelCase							:     Any	=		subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。''')
             self.assertListEqual(a__      ,							['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。'''])

             _lowerCamelCase							:     str	=		subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは''')
             self.assertListEqual(a__      ,							['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは'''])



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese''')

             _lowerCamelCase							:     int	=		tokenizer.encode('''ありがとう。'''      ,							add_special_tokens=a__)
             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.encode('''どういたしまして。'''      ,							add_special_tokens=a__)

             _lowerCamelCase							:     int	=		tokenizer.build_inputs_with_special_tokens(a__)
             _lowerCamelCase							:     List[str]	=		tokenizer.build_inputs_with_special_tokens(a__      ,							a__)

             # 2 is for "[CLS]", 3 is for "[SEP]"
             assert encoded_sentence == [2] + text + [3]
             assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 			__A       (      lowerCamelCase__ ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      BertJapaneseTokenizer
      UpperCAmelCase__						=      False


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             _lowerCamelCase							:     Dict	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

             _lowerCamelCase							:     Optional[int]	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''vocab_file'''])
             with open(self.vocab_file      ,							'''w'''      ,							encoding='''utf-8''') as vocab_writer:
                    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


      def 				__snake_case			(		self      ,							**a__):



             """simple docstring"""





             return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,							subword_tokenizer_type='''character'''      ,							**a__)


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		'''こんにちは、世界。 \nこんばんは、世界。'''
             _lowerCamelCase							:     int	=		'''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
             return input_text, output_text


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             pass  # TODO add if relevant


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     int	=		self.tokenizer_class(self.vocab_file      ,							subword_tokenizer_type='''character''')

             _lowerCamelCase							:     Optional[int]	=		tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。''')
             self.assertListEqual(
                 a__      ,							['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。'''])
             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(a__)      ,							[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

             _lowerCamelCase							:     List[str]	=		{}
             for i, token in enumerate(a__):
                    _lowerCamelCase							:     List[Any]	=		i
             _lowerCamelCase							:     Union[str, Any]	=		CharacterTokenizer(vocab=a__      ,							unk_token='''[UNK]''')

             self.assertListEqual(tokenizer.tokenize('''''')      ,							[])

             self.assertListEqual(tokenizer.tokenize('''こんにちは''')      ,							['''こ''', '''ん''', '''に''', '''ち''', '''は'''])

             self.assertListEqual(tokenizer.tokenize('''こんにちほ''')      ,							['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]'''])



      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Dict	=		self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char''')

             _lowerCamelCase							:     List[str]	=		tokenizer.encode('''ありがとう。'''      ,							add_special_tokens=a__)
             _lowerCamelCase							:     Optional[int]	=		tokenizer.encode('''どういたしまして。'''      ,							add_special_tokens=a__)

             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.build_inputs_with_special_tokens(a__)
             _lowerCamelCase							:     int	=		tokenizer.build_inputs_with_special_tokens(a__      ,							a__)

             # 2 is for "[CLS]", 3 is for "[SEP]"
             assert encoded_sentence == [2] + text + [3]
             assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 			__A       (      unittest.TestCase  ):




      """simple docstring"""







      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		'''cl-tohoku/bert-base-japanese'''
             _lowerCamelCase							:     Optional[int]	=		AutoTokenizer.from_pretrained(a__)
             self.assertIsInstance(a__      ,							a__)







class 			__A       (      unittest.TestCase  ):




      """simple docstring"""







      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		'''cl-tohoku/bert-base-japanese'''
             with self.assertLogs('''transformers'''      ,							level='''WARNING''') as cm:
                    BertTokenizer.from_pretrained(a__)
                    self.assertTrue(
                        cm.records[0].message.startswith(
                            '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                            ''' is called from.'''))
             _lowerCamelCase							:     List[Any]	=		'''bert-base-cased'''
             with self.assertLogs('''transformers'''      ,							level='''WARNING''') as cm:
                    BertJapaneseTokenizer.from_pretrained(a__)
                    self.assertTrue(
                        cm.records[0].message.startswith(
                            '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                            ''' is called from.'''))

