






import argparse
import os
import re


A__			:    Dict         =				'src/transformers/models/auto'


# re pattern that matches mapping introductions:
#    SUPER_MODEL_MAPPING_NAMES = OrderedDict or SUPER_MODEL_MAPPING = OrderedDict
A__			:    List[Any]         =				re.compile(r'[A-Z_]+_MAPPING(\s+|_[A-Z_]+\s+)=\s+OrderedDict')
# re pattern that matches identifiers in mappings
A__			:    Union[str, Any]         =				re.compile(r'\s*\(\s*"(\S[^"]+)"')






def 						a    (		lowerCamelCase_	,     lowerCamelCase_ = False       ):






				'''simple docstring'''
				with open(lowerCamelCase_	,     '''r'''	,     encoding='''utf-8'''       ) as f:
								lowercase__     				=    f.read()

				lowercase__     				=    content.split('''\n'''       )
				lowercase__     				=    []
				lowercase__     				=    0
				while line_idx < len(lowerCamelCase_       ):
								if _re_intro_mapping.search(lines[line_idx]       ) is not None:
												lowercase__     				=    len(re.search(r'''^(\s*)\S'''	,     lines[line_idx]       ).groups()[0]       ) + 8
												# Start of a new mapping!
												while not lines[line_idx].startswith(''' ''' * indent + '''('''       ):
																new_lines.append(lines[line_idx]       )
																line_idx += 1

												lowercase__     				=    []
												while lines[line_idx].strip() != "]":
																# Blocks either fit in one line or not
																if lines[line_idx].strip() == "(":
																				lowercase__     				=    line_idx
																				while not lines[line_idx].startswith(''' ''' * indent + ''')'''       ):
																								line_idx += 1
																				blocks.append('''\n'''.join(lines[start_idx : line_idx + 1]       )       )
																else:
																				blocks.append(lines[line_idx]       )
																line_idx += 1

												# Sort blocks by their identifiers
												lowercase__     				=    sorted(lowerCamelCase_	,     key=lambda lowerCamelCase_       : _re_identifier.search(lowerCamelCase_       ).groups()[0]       )
												new_lines += blocks
								else:
												new_lines.append(lines[line_idx]       )
												line_idx += 1

				if overwrite:
								with open(lowerCamelCase_	,     '''w'''	,     encoding='''utf-8'''       ) as f:
												f.write('''\n'''.join(lowerCamelCase_       )       )
				elif "\n".join(lowerCamelCase_       ) != content:
								return True






def 						a    (		lowerCamelCase_ = False       ):






				'''simple docstring'''
				lowercase__     				=    [os.path.join(lowerCamelCase_	,     lowerCamelCase_       ) for f in os.listdir(lowerCamelCase_       ) if f.endswith('''.py'''       )]
				lowercase__     				=    [sort_auto_mapping(lowerCamelCase_	,     overwrite=lowerCamelCase_       ) for fname in fnames]

				if not overwrite and any(lowerCamelCase_       ):
								lowercase__     				=    [f for f, d in zip(lowerCamelCase_	,     lowerCamelCase_       ) if d]
								raise ValueError(
								    F"""The following files have auto mappings that need sorting: {', '.join(lowerCamelCase_       )}. Run `make style` to fix"""
								    ''' this.'''       )


if __name__ == "__main__":
							A__			:    Union[str, Any]         =				argparse.ArgumentParser()
							parser.add_argument('--check_only', action='store_true', help='Whether to only check or fix style.')
							A__			:    Dict         =				parser.parse_args()

							sort_all_auto_mappings(not args.check_only)












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


A__			:    Dict         =				logging.get_logger(__name__)

A__			:    Dict         =				{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__			:    Optional[int]         =				{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

A__			:    List[str]         =				{
    'bert-base-uncased': 5_12,
    'bert-large-uncased': 5_12,
    'bert-base-cased': 5_12,
    'bert-large-cased': 5_12,
    'bert-base-multilingual-uncased': 5_12,
    'bert-base-multilingual-cased': 5_12,
    'bert-base-chinese': 5_12,
    'bert-base-german-cased': 5_12,
    'bert-large-uncased-whole-word-masking': 5_12,
    'bert-large-cased-whole-word-masking': 5_12,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 5_12,
    'bert-large-cased-whole-word-masking-finetuned-squad': 5_12,
    'bert-base-cased-finetuned-mrpc': 5_12,
    'bert-base-german-dbmdz-cased': 5_12,
    'bert-base-german-dbmdz-uncased': 5_12,
    'TurkuNLP/bert-base-finnish-cased-v1': 5_12,
    'TurkuNLP/bert-base-finnish-uncased-v1': 5_12,
    'wietsedv/bert-base-dutch-cased': 5_12,
}

A__			:    Optional[int]         =				{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}
class    _UpperCAmelCase      ( A__			):

				"""simple docstring"""




				lowercase__       							= VOCAB_FILES_NAMES
				lowercase__       							= PRETRAINED_VOCAB_FILES_MAP
				lowercase__       							= PRETRAINED_INIT_CONFIGURATION
				lowercase__       							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowercase__       							= BertTokenizer




				def __init__(  self		:					Any,						lowerCamelCase		:					Optional[Any]=None,						lowerCamelCase		:					Any=None,						lowerCamelCase		:					Tuple=True,						lowerCamelCase		:					Dict="[UNK]",						lowerCamelCase		:					Any="[SEP]",						lowerCamelCase		:					List[Any]="[PAD]",						lowerCamelCase		:					Optional[Any]="[CLS]",						lowerCamelCase		:					Dict="[MASK]",						lowerCamelCase		:					List[Any]=True,						lowerCamelCase		:					Tuple=None,						**lowerCamelCase		:					Dict,						):

								'''simple docstring'''
								super().__init__(
								    lowerCamelCase,						tokenizer_file=lowerCamelCase,						do_lower_case=lowerCamelCase,						unk_token=lowerCamelCase,						sep_token=lowerCamelCase,						pad_token=lowerCamelCase,						cls_token=lowerCamelCase,						mask_token=lowerCamelCase,						tokenize_chinese_chars=lowerCamelCase,						strip_accents=lowerCamelCase,						**lowerCamelCase,						)

								lowercase__     				=    json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
								if (
								    normalizer_state.get('''lowercase''',						lowerCamelCase						) != do_lower_case
								    or normalizer_state.get('''strip_accents''',						lowerCamelCase						) != strip_accents
								    or normalizer_state.get('''handle_chinese_chars''',						lowerCamelCase						) != tokenize_chinese_chars
								):
												lowercase__     				=    getattr(lowerCamelCase,						normalizer_state.pop('''type'''						)						)
												lowercase__     				=    do_lower_case
												lowercase__     				=    strip_accents
												lowercase__     				=    tokenize_chinese_chars
												lowercase__     				=    normalizer_class(**lowerCamelCase						)

								lowercase__     				=    do_lower_case




				def        lowercase__     (  self		:					Any,						lowerCamelCase		:					List[Any],						lowerCamelCase		:					Dict=None						):

								'''simple docstring'''
								lowercase__     				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
												output += token_ids_a + [self.sep_token_id]

								return output




				def        lowercase__     (  self		:					List[Any],						lowerCamelCase		:					List[int],						lowerCamelCase		:					Optional[List[int]] = None						):

								'''simple docstring'''
								lowercase__     				=    [self.sep_token_id]
								lowercase__     				=    [self.cls_token_id]
								if token_ids_a is None:
												return len(cls + token_ids_a + sep						) * [0]
								return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




				def        lowercase__     (  self		:					Any,						lowerCamelCase		:					str,						lowerCamelCase		:					Optional[str] = None						):

								'''simple docstring'''
								lowercase__     				=    self._tokenizer.model.save(lowerCamelCase,						name=lowerCamelCase						)
								return tuple(lowerCamelCase						)




