





import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

lowerCAmelCase_						   =    {
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

lowerCAmelCase_						   =    {
    """abeja/gpt-neox-japanese-2.7b""": 2048,
}


def     lowerCamelCase_	(      lowerCAmelCase:	List[Any]    ,							lowerCAmelCase:	List[str]				)->		str:





  with open(__snake_case    ,							'r'    ,							encoding='utf-8'				) as f:
    _snake_case   :				int														=	json.loads(f.read()				)

  _snake_case   :				Optional[int]														=	collections.OrderedDict()
  _snake_case   :				Optional[int]														=	collections.OrderedDict()
  _snake_case   :				Dict														=	collections.OrderedDict()
  with open(__snake_case    ,							'r'    ,							encoding='utf-8'				) as f:
    _snake_case   :				Optional[Any]														=	f.readlines()
  _snake_case   :				Optional[int]														=	[[t.rstrip('\n'				)] if (t == ',' or ',' not in t) else t.rstrip('\n'				).split(','				) for t in token]
  for idx, b in enumerate(__snake_case				):
    _snake_case   :				List[Any]														=	b
    _snake_case   :				Tuple														=	idx
    for wd in b:
      _snake_case   :				int														=	idx

  return vocab, raw_vocab, ids_to_tokens, emoji



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       List[str]      					=VOCAB_FILES_NAMES
  a_							:       Any      					=PRETRAINED_VOCAB_FILES_MAP
  a_							:       List[str]      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_							:       Optional[int]      					=["""input_ids""", """attention_mask"""]







  def __init__(      self      :					Optional[int]      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					Union[str, Any]="<|endoftext|>"      ,				UpperCamelCase      :					str="<|endoftext|>"      ,				UpperCamelCase      :					Tuple="<|startoftext|>"      ,				UpperCamelCase      :					List[str]="<|endoftext|>"      ,				UpperCamelCase      :					Union[str, Any]=False      ,				**UpperCamelCase      :					Any      ,				):


    '''simple docstring'''





    super().__init__(
        unk_token=__a      ,				pad_token=__a      ,				bos_token=__a      ,				eos_token=__a      ,				do_clean_text=__a      ,				**__a      ,				)
    if not os.path.isfile(__a							):
      raise ValueError(
          f"""Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained"""
          ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'							)
    if not os.path.isfile(__a							):
      raise ValueError(
          f"""Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google"""
          ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'							)
    _snake_case   :				Dict														=	do_clean_text
    _snake_case							,					_snake_case							,					_snake_case							,					_snake_case   :				List[str]														=	load_vocab_and_emoji(__a      ,				__a							)
    _snake_case   :				Optional[int]														=	SubWordJapaneseTokenizer(
        vocab=self.vocab      ,				ids_to_tokens=self.ids_to_tokens      ,				emoji=self.emoji							)







  @property
  def UpperCamelCase_ (      self      :					List[Any]							):


    '''simple docstring'''





    return len(self.raw_vocab							)







  def UpperCamelCase_ (      self      :					Tuple							):


    '''simple docstring'''





    return dict(self.raw_vocab      ,				**self.added_tokens_encoder							)







  def UpperCamelCase_ (      self      :					List[str]      ,				UpperCamelCase      :					Any							):


    '''simple docstring'''





    return self.subword_tokenizer.tokenize(__a      ,				clean=self.do_clean_text							)







  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					Dict							):


    '''simple docstring'''





    return self.vocab.get(__a      ,				self.vocab.get(self.unk_token							)							)







  def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					Dict							):


    '''simple docstring'''





    return self.subword_tokenizer.convert_id_to_token(__a							)







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					Optional[int]							):


    '''simple docstring'''





    _snake_case   :				int														=	''.join(__a							).strip()
    return out_string







  def UpperCamelCase_ (      self      :					int      ,				UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	[]
    for is_user, text in conversation.iter_texts():
      input_ids.extend(self.encode(__a      ,				add_special_tokens=__a							) + [self.eos_token_id]							)

    if len(__a							) > self.model_max_length:
      _snake_case   :				Optional[int]														=	input_ids[-self.model_max_length :]
    return input_ids







  def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					Any = None							):


    '''simple docstring'''





    _snake_case   :				Union[str, Any]														=	0
    if os.path.isdir(__a							):
      _snake_case   :				str														=	os.path.join(
          __a      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
      _snake_case   :				int														=	os.path.join(
          __a      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']							)
    else:
      _snake_case   :				Dict														=	(
          (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
      )
      _snake_case   :				Union[str, Any]														=	(
          (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
      )
    with open(__a      ,				'w'      ,				encoding='utf-8'							) as writer:
      for token_index, token in self.ids_to_tokens.items():
        if index != token_index:
          logger.warning(
              f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
              ' Please check that the vocabulary is not corrupted!'							)
          _snake_case   :				Tuple														=	token_index
        writer.write(','.join(__a							) + '\n'							)
        index += 1
    with open(__a      ,				'w'      ,				encoding='utf-8'							) as writer:
      json.dump(self.emoji      ,				__a							)
    return vocab_file, emoji_file




class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''







  def __init__(      self      :					int      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				List[str]														=	vocab  # same as swe
    _snake_case   :				Optional[int]														=	ids_to_tokens  # same as bpe
    _snake_case   :				Any														=	emoji
    _snake_case   :				Tuple														=	np.max([len(__a							) for w in self.vocab.keys()]							)
    _snake_case   :				Optional[int]														=	re.compile(R'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'							)
    _snake_case   :				List[str]														=	re.compile(R'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'							)
    _snake_case   :				List[str]														=	re.compile(R'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'							)
    _snake_case   :				Union[str, Any]														=	re.compile(
        R'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'							)
    _snake_case   :				List[str]														=	re.compile(
        R'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'							)
    _snake_case   :				int														=	re.compile(
        R'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'							)
    _snake_case   :				Optional[Any]														=	'─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
    _snake_case   :				Optional[Any]														=	'▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
    _snake_case   :				List[str]														=	str.maketrans({k: '<BLOCK>' for k in keisen + blocks}							)







  def __len__(      self      :					Dict							):


    '''simple docstring'''





    return len(self.ids_to_tokens							)







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					Optional[Any]							):


    '''simple docstring'''





    _snake_case   :				Tuple														=	self.content_repattera.sub('<URL>'      ,				__a							)
    _snake_case   :				Union[str, Any]														=	self.content_repattera.sub('<EMAIL>'      ,				__a							)
    _snake_case   :				Any														=	self.content_repattera.sub('<TEL>'      ,				__a							)
    _snake_case   :				Optional[int]														=	self.content_repattera.sub('<DATE>'      ,				__a							)
    _snake_case   :				Any														=	self.content_repattera.sub('<DATE>'      ,				__a							)
    _snake_case   :				Tuple														=	self.content_repattera.sub('<PRICE>'      ,				__a							)
    _snake_case   :				Optional[Any]														=	content.translate(self.content_transa							)
    while "<BLOCK><BLOCK>" in content:
      _snake_case   :				Union[str, Any]														=	content.replace('<BLOCK><BLOCK>'      ,				'<BLOCK>'							)
    return content







  def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					List[str]      ,				UpperCamelCase      :					Union[str, Any]=False							):


    '''simple docstring'''





    _snake_case   :				List[str]														=	text.replace(' '      ,				'<SP>'							)
    _snake_case   :				Optional[int]														=	text.replace('　'      ,				'<SP>'							)
    _snake_case   :				List[Any]														=	text.replace('\r\n'      ,				'<BR>'							)
    _snake_case   :				List[Any]														=	text.replace('\n'      ,				'<BR>'							)
    _snake_case   :				Optional[Any]														=	text.replace('\r'      ,				'<BR>'							)
    _snake_case   :				Dict														=	text.replace('\t'      ,				'<TAB>'							)
    _snake_case   :				Optional[Any]														=	text.replace('—'      ,				'ー'							)
    _snake_case   :				List[Any]														=	text.replace('−'      ,				'ー'							)
    for k, v in self.emoji["emoji"].items():
      if k in text:
        _snake_case   :				Tuple														=	text.replace(__a      ,				__a							)
    if clean:
      _snake_case   :				Tuple														=	self.clean_text(__a							)

    def check_simbol(UpperCamelCase      :					List[Any]							):
      _snake_case   :				Union[str, Any]														=	x.encode()
      if len(__a							) == 1 and len(__a							) == 2:
        _snake_case   :				Union[str, Any]														=	(int(e[0]							) << 8) + int(e[1]							)
        if (
            (c >= 0xc2a1 and c <= 0xc2bf)
            or (c >= 0xc780 and c <= 0xc783)
            or (c >= 0xcab9 and c <= 0xcbbf)
            or (c >= 0xcc80 and c <= 0xcda2)
        ):
          return True
      return False

    def checkuae(UpperCamelCase      :					Tuple							):
      _snake_case   :				Union[str, Any]														=	x.encode()
      if len(__a							) == 1 and len(__a							) == 3:
        _snake_case   :				Tuple														=	(int(e[0]							) << 16) + (int(e[1]							) << 8) + int(e[2]							)
        if c >= 0xe28080 and c <= 0xe2b07f:
          return True
      return False

    _snake_case   :				int														=	0
    _snake_case   :				Optional[int]														=	[]
    while pos < len(__a							):
      _snake_case   :				List[Any]														=	min(len(__a							)      ,				pos + self.maxlen + 1							) if text[pos] == '<' else pos + 3
      _snake_case   :				Tuple														=	[]  # (token_id, token, pos)
      for e in range(__a      ,				__a      ,				-1							):
        _snake_case   :				List[Any]														=	text[pos:e]
        if wd in self.vocab:
          if wd[0] == "<" and len(__a							) > 2:
            _snake_case   :				Optional[Any]														=	[(self.vocab[wd], wd, e)]
            break
          else:
            candidates.append((self.vocab[wd], wd, e)							)
      if len(__a							) > 0:
        # the smallest token_id is adopted
        _snake_case							,					_snake_case							,					_snake_case   :				List[str]														=	sorted(__a      ,				key=lambda UpperCamelCase							: x[0]							)[0]
        result.append(__a							)
        _snake_case   :				str														=	e
      else:
        _snake_case   :				Dict														=	pos + 1
        _snake_case   :				Dict														=	text[pos:end]
        if check_simbol(__a							):
          result.append('<KIGOU>'							)
        elif checkuae(__a							):
          result.append('<U2000U2BFF>'							)
        else:
          for i in wd.encode('utf-8'							):
            result.append('<|byte%d|>' % i							)
        _snake_case   :				Any														=	end
    return result







  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					Union[str, Any]="\n"							):


    '''simple docstring'''





    _snake_case   :				int														=	[]
    _snake_case   :				int														=	[]
    _snake_case   :				str														=	self.ids_to_tokens[index][0]
    if word[:6] == "<|byte" and word[-2:] == "|>":
      byte_tokens.append(int(word[6:-2]							)							)
    else:
      if len(__a							) > 0:
        words.append(bytearray(__a							).decode('utf-8'      ,				errors='replace'							)							)
        _snake_case   :				str														=	[]
      if word[:7] == "<|emoji" and word[-2:] == "|>":
        words.append(self.emoji['emoji_inv'][word]							)
      elif word == "<SP>":
        words.append(' '							)
      elif word == "<BR>":
        words.append(__a							)
      elif word == "<TAB>":
        words.append('\t'							)
      elif word == "<BLOCK>":
        words.append('▀'							)
      elif word == "<KIGOU>":
        words.append('ǀ'							)
      elif word == "<U2000U2BFF>":
        words.append('‖'							)
      else:
        words.append(__a							)
    if len(__a							) > 0:
      words.append(bytearray(__a							).decode('utf-8'      ,				errors='replace'							)							)
    _snake_case   :				Any														=	''.join(__a							)
    return text











# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


lowerCAmelCase_						   =    """Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"""


def     lowerCamelCase_	(      )->		Tuple:
  _snake_case   :				int														=	_ask_options(
      'In which compute environment are you running?'    ,							['This machine', 'AWS (Amazon SageMaker)']    ,							_convert_compute_environment    ,							)
  if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
    _snake_case   :				int														=	get_sagemaker_input()
  else:
    _snake_case   :				Any														=	get_cluster_input()
  return config


def     lowerCamelCase_	(      lowerCAmelCase:	str=None				)->		Any:
  if subparsers is not None:
    _snake_case   :				List[Any]														=	subparsers.add_parser('config'    ,							description=lowerCAmelCase				)
  else:
    _snake_case   :				Dict														=	argparse.ArgumentParser('Accelerate config command'    ,							description=lowerCAmelCase				)

  parser.add_argument(
      '--config_file'    ,							default=lowerCAmelCase    ,							help=(
          'The path to use to store the config file. Will default to a file named default_config.yaml in the cache '
          'location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '
          'such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '
          'with \'huggingface\'.'
      )    ,							)

  if subparsers is not None:
    parser.set_defaults(func=lowerCAmelCase				)
  return parser


def     lowerCamelCase_	(      lowerCAmelCase:	Any				)->		Any:
  _snake_case   :				Dict														=	get_user_input()
  if args.config_file is not None:
    _snake_case   :				List[str]														=	args.config_file
  else:
    if not os.path.isdir(lowerCAmelCase				):
      os.makedirs(lowerCAmelCase				)
    _snake_case   :				Union[str, Any]														=	default_yaml_config_file

  if config_file.endswith('.json'				):
    config.to_json_file(lowerCAmelCase				)
  else:
    config.to_yaml_file(lowerCAmelCase				)
  print(F"""accelerate configuration saved at {config_file}"""				)


def     lowerCamelCase_	(      )->		Dict:
  _snake_case   :				List[str]														=	config_command_parser()
  _snake_case   :				str														=	parser.parse_args()
  config_command(lowerCAmelCase				)


if __name__ == "__main__":
      main()




