





'''simple docstring'''




import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


_A:					Any							=				logging.get_logger(__name__)

_A:					Union[str, Any]							=				{"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

_A:					Optional[Any]							=				{
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

_A:					Optional[int]							=				{
    """abeja/gpt-neox-japanese-2.7b""": 2_048,
}







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase    )->     Any:
 with open(_lowerCAmelCase       ,    'r'       ,    encoding='utf-8'    ) as f:
  __UpperCAmelCase					=   json.loads(f.read()    )

 __UpperCAmelCase					=   collections.OrderedDict()
 __UpperCAmelCase					=   collections.OrderedDict()
 __UpperCAmelCase					=   collections.OrderedDict()
 with open(_lowerCAmelCase       ,    'r'       ,    encoding='utf-8'    ) as f:
  __UpperCAmelCase					=   f.readlines()
 __UpperCAmelCase					=   [[t.rstrip('\n'    )] if (t == ',' or ',' not in t) else t.rstrip('\n'    ).split(','    ) for t in token]
 for idx, b in enumerate(_lowerCAmelCase    ):
  __UpperCAmelCase					=   b
  __UpperCAmelCase					=   idx
  for wd in b:
   __UpperCAmelCase					=   idx

 return vocab, raw_vocab, ids_to_tokens, emoji






class UpperCAmelCase							(		UpperCAmelCase_							):
    _A :			Tuple						   =       VOCAB_FILES_NAMES
    _A :			List[str]						   =       PRETRAINED_VOCAB_FILES_MAP
    _A :			Any						   =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _A :			Any						   =       ["""input_ids""", """attention_mask"""]



    def __init__(  self     ,			__A     ,			__A     ,			__A="<|endoftext|>"     ,			__A="<|endoftext|>"     ,			__A="<|startoftext|>"     ,			__A="<|endoftext|>"     ,			__A=False     ,			**__A     ,			):
     super().__init__(
         unk_token=__A     ,			pad_token=__A     ,			bos_token=__A     ,			eos_token=__A     ,			do_clean_text=__A     ,			**__A     ,			)
     if not os.path.isfile(__A					):
      raise ValueError(
          f'Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'
          ' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'					)
     if not os.path.isfile(__A					):
      raise ValueError(
          f'Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'
          ' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'					)
     __UpperCAmelCase					=   do_clean_text
     __UpperCAmelCase					,    __UpperCAmelCase					,    __UpperCAmelCase					,    __UpperCAmelCase					=   load_vocab_and_emoji(__A     ,			__A					)
     __UpperCAmelCase					=   SubWordJapaneseTokenizer(
         vocab=self.vocab     ,			ids_to_tokens=self.ids_to_tokens     ,			emoji=self.emoji					)



    @property
    def 					__lowerCamelCase						(  self					):
     # self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
     return len(self.raw_vocab					)



    def 					__lowerCamelCase						(  self					):
     return dict(self.raw_vocab     ,			**self.added_tokens_encoder					)



    def 					__lowerCamelCase						(  self     ,			__A					):
     return self.subword_tokenizer.tokenize(__A     ,			clean=self.do_clean_text					)



    def 					__lowerCamelCase						(  self     ,			__A					):
     return self.vocab.get(__A     ,			self.vocab.get(self.unk_token					)					)



    def 					__lowerCamelCase						(  self     ,			__A					):
     return self.subword_tokenizer.convert_id_to_token(__A					)



    def 					__lowerCamelCase						(  self     ,			__A					):
     __UpperCAmelCase					=   ''.join(__A					).strip()
     return out_string



    def 					__lowerCamelCase						(  self     ,			__A					):
     __UpperCAmelCase					=   []
     for is_user, text in conversation.iter_texts():
      input_ids.extend(self.encode(__A     ,			add_special_tokens=__A					) + [self.eos_token_id]					)

     if len(__A					) > self.model_max_length:
      __UpperCAmelCase					=   input_ids[-self.model_max_length :]
     return input_ids




    def 					__lowerCamelCase						(  self     ,			__A     ,			__A = None					):
     __UpperCAmelCase					=   0
     if os.path.isdir(__A					):
      __UpperCAmelCase					=   os.path.join(
          __A     ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
      __UpperCAmelCase					=   os.path.join(
          __A     ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['emoji_file']					)
     else:
      __UpperCAmelCase					=   (
          (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['vocab_file']
      )
      __UpperCAmelCase					=   (
          (filename_prefix + '-' if filename_prefix else '') + save_directory + VOCAB_FILES_NAMES['emoji_file']
      )
     with open(__A     ,			'w'     ,			encoding='utf-8'					) as writer:
      for token_index, token in self.ids_to_tokens.items():
       if index != token_index:
        logger.warning(
            f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
            ' Please check that the vocabulary is not corrupted!'					)
        __UpperCAmelCase					=   token_index
       writer.write(','.join(__A					) + '\n'					)
       index += 1
     with open(__A     ,			'w'     ,			encoding='utf-8'					) as writer:
      json.dump(self.emoji     ,			__A					)
     return vocab_file, emoji_file






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A					):
     __UpperCAmelCase					=   vocab  # same as swe
     __UpperCAmelCase					=   ids_to_tokens  # same as bpe
     __UpperCAmelCase					=   emoji
     __UpperCAmelCase					=   np.max([len(__A					) for w in self.vocab.keys()]					)
     __UpperCAmelCase					=   re.compile(r'(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'					)
     __UpperCAmelCase					=   re.compile(r'[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'					)
     __UpperCAmelCase					=   re.compile(r'[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'					)
     __UpperCAmelCase					=   re.compile(
         r'([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'					)
     __UpperCAmelCase					=   re.compile(
         r'(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'					)
     __UpperCAmelCase					=   re.compile(
         r'((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'					)
     __UpperCAmelCase					=   '─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'
     __UpperCAmelCase					=   '▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'
     __UpperCAmelCase					=   str.maketrans({k: '<BLOCK>' for k in keisen + blocks}					)



    def __len__(  self					):
     return len(self.ids_to_tokens					)



    def 					__lowerCamelCase						(  self     ,			__A					):
     __UpperCAmelCase					=   self.content_repattera.sub('<URL>'     ,			__A					)
     __UpperCAmelCase					=   self.content_repattera.sub('<EMAIL>'     ,			__A					)
     __UpperCAmelCase					=   self.content_repattera.sub('<TEL>'     ,			__A					)
     __UpperCAmelCase					=   self.content_repattera.sub('<DATE>'     ,			__A					)
     __UpperCAmelCase					=   self.content_repattera.sub('<DATE>'     ,			__A					)
     __UpperCAmelCase					=   self.content_repattera.sub('<PRICE>'     ,			__A					)
     __UpperCAmelCase					=   content.translate(self.content_transa					)
     while "<BLOCK><BLOCK>" in content:
      __UpperCAmelCase					=   content.replace('<BLOCK><BLOCK>'     ,			'<BLOCK>'					)
     return content



    def 					__lowerCamelCase						(  self     ,			__A     ,			__A=False					):
     __UpperCAmelCase					=   text.replace(' '     ,			'<SP>'					)
     __UpperCAmelCase					=   text.replace('　'     ,			'<SP>'					)
     __UpperCAmelCase					=   text.replace('\r\n'     ,			'<BR>'					)
     __UpperCAmelCase					=   text.replace('\n'     ,			'<BR>'					)
     __UpperCAmelCase					=   text.replace('\r'     ,			'<BR>'					)
     __UpperCAmelCase					=   text.replace('\t'     ,			'<TAB>'					)
     __UpperCAmelCase					=   text.replace('—'     ,			'ー'					)
     __UpperCAmelCase					=   text.replace('−'     ,			'ー'					)
     for k, v in self.emoji["emoji"].items():
      if k in text:
       __UpperCAmelCase					=   text.replace(__A     ,			__A					)
     if clean:
      __UpperCAmelCase					=   self.clean_text(__A					)

     def check_simbol(__A					):
      __UpperCAmelCase					=   x.encode()
      if len(__A					) == 1 and len(__A					) == 2:
       __UpperCAmelCase					=   (int(e[0]					) << 8) + int(e[1]					)
       if (
           (c >= 0xC_2_A_1 and c <= 0xC_2_B_F)
           or (c >= 0xC_7_8_0 and c <= 0xC_7_8_3)
           or (c >= 0xC_A_B_9 and c <= 0xC_B_B_F)
           or (c >= 0xC_C_8_0 and c <= 0xC_D_A_2)
       ):
        return True
      return False

     def checkuae(__A					):
      __UpperCAmelCase					=   x.encode()
      if len(__A					) == 1 and len(__A					) == 3:
       __UpperCAmelCase					=   (int(e[0]					) << 16) + (int(e[1]					) << 8) + int(e[2]					)
       if c >= 0xE_2_8_0_8_0 and c <= 0xE_2_B_0_7_F:
        return True
      return False

     __UpperCAmelCase					=   0
     __UpperCAmelCase					=   []
     while pos < len(__A					):
      __UpperCAmelCase					=   min(len(__A					)     ,			pos + self.maxlen + 1					) if text[pos] == '<' else pos + 3
      __UpperCAmelCase					=   []  # (token_id, token, pos)
      for e in range(__A     ,			__A     ,			-1					):
       __UpperCAmelCase					=   text[pos:e]
       if wd in self.vocab:
        if wd[0] == "<" and len(__A					) > 2:
         __UpperCAmelCase					=   [(self.vocab[wd], wd, e)]
         break
        else:
         candidates.append((self.vocab[wd], wd, e)					)
      if len(__A					) > 0:
       # the smallest token_id is adopted
       __UpperCAmelCase					,    __UpperCAmelCase					,    __UpperCAmelCase					=   sorted(__A     ,			key=lambda __A					: x[0]					)[0]
       result.append(__A					)
       __UpperCAmelCase					=   e
      else:
       __UpperCAmelCase					=   pos + 1
       __UpperCAmelCase					=   text[pos:end]
       if check_simbol(__A					):
        result.append('<KIGOU>'					)
       elif checkuae(__A					):
        result.append('<U2000U2BFF>'					)
       else:
        for i in wd.encode('utf-8'					):
         result.append('<|byte%d|>' % i					)
       __UpperCAmelCase					=   end
     return result




    def 					__lowerCamelCase						(  self     ,			__A     ,			__A="\n"					):
     __UpperCAmelCase					=   []
     __UpperCAmelCase					=   []
     __UpperCAmelCase					=   self.ids_to_tokens[index][0]
     if word[:6] == "<|byte" and word[-2:] == "|>":
      byte_tokens.append(int(word[6:-2]					)					)
     else:
      if len(__A					) > 0:
       words.append(bytearray(__A					).decode('utf-8'     ,			errors='replace'					)					)
       __UpperCAmelCase					=   []
      if word[:7] == "<|emoji" and word[-2:] == "|>":
       words.append(self.emoji['emoji_inv'][word]					)
      elif word == "<SP>":
       words.append(' '					)
      elif word == "<BR>":
       words.append(__A					)
      elif word == "<TAB>":
       words.append('\t'					)
      elif word == "<BLOCK>":
       words.append('▀'					)
      elif word == "<KIGOU>":
       words.append('ǀ'					)
      elif word == "<U2000U2BFF>":
       words.append('‖'					)
      else:
       words.append(__A					)
     if len(__A					) > 0:
      words.append(bytearray(__A					).decode('utf-8'     ,			errors='replace'					)					)
     __UpperCAmelCase					=   ''.join(__A					)
     return text












'''simple docstring'''




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path

import torch

from ...utils import is_npu_available, is_xpu_available
from .config_args import ClusterConfig, default_json_config_file
from .config_utils import SubcommandHelpFormatter


_A:					str							=				"""Create a default config file for Accelerate with only a few flags set."""







def _lowerCAmelCase     (      _lowerCAmelCase="no"       ,    _lowerCAmelCase = default_json_config_file       ,    _lowerCAmelCase = False    )->     List[Any]:
 __UpperCAmelCase					=   Path(_lowerCAmelCase    )
 path.parent.mkdir(parents=_lowerCAmelCase       ,    exist_ok=_lowerCAmelCase    )
 if path.exists():
  print(
      F'Configuration already exists at {save_location}, will not override. Run `accelerate config` manually or pass a different `save_location`.'    )
  return False
 __UpperCAmelCase					=   mixed_precision.lower()
 if mixed_precision not in ["no", "fp16", "bf16", "fp8"]:
  raise ValueError(
      F'`mixed_precision` should be one of \'no\', \'fp16\', \'bf16\', or \'fp8\'. Received {mixed_precision}'    )
 __UpperCAmelCase					=   {
     'compute_environment': 'LOCAL_MACHINE',
     'mixed_precision': mixed_precision,
 }
 if torch.cuda.is_available():
  __UpperCAmelCase					=   torch.cuda.device_count()
  __UpperCAmelCase					=   num_gpus
  __UpperCAmelCase					=   False
  if num_gpus > 1:
   __UpperCAmelCase					=   'MULTI_GPU'
  else:
   __UpperCAmelCase					=   'NO'
 elif is_xpu_available() and use_xpu:
  __UpperCAmelCase					=   torch.xpu.device_count()
  __UpperCAmelCase					=   num_xpus
  __UpperCAmelCase					=   False
  if num_xpus > 1:
   __UpperCAmelCase					=   'MULTI_XPU'
  else:
   __UpperCAmelCase					=   'NO'
 elif is_npu_available():
  __UpperCAmelCase					=   torch.npu.device_count()
  __UpperCAmelCase					=   num_npus
  __UpperCAmelCase					=   False
  if num_npus > 1:
   __UpperCAmelCase					=   'MULTI_NPU'
  else:
   __UpperCAmelCase					=   'NO'
 else:
  __UpperCAmelCase					=   0
  __UpperCAmelCase					=   True
  __UpperCAmelCase					=   1
  __UpperCAmelCase					=   'NO'
 __UpperCAmelCase					=   ClusterConfig(**_lowerCAmelCase    )
 config.to_json_file(_lowerCAmelCase    )
 return path







def _lowerCAmelCase     (      _lowerCAmelCase       ,    _lowerCAmelCase    )->     List[str]:
 __UpperCAmelCase					=   parser.add_parser('default'       ,    parents=_lowerCAmelCase       ,    help=_lowerCAmelCase       ,    formatter_class=_lowerCAmelCase    )
 parser.add_argument(
     '--config_file'       ,    default=_lowerCAmelCase       ,    help=(
         'The path to use to store the config file. Will default to a file named default_config.yaml in the cache '
         'location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '
         'such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '
         'with \'huggingface\'.'
     )       ,    dest='save_location'       ,    )

 parser.add_argument(
     '--mixed_precision'       ,    choices=['no', 'fp16', 'bf16']       ,    type=_lowerCAmelCase       ,    help='Whether or not to use mixed precision training. '
     'Choose between FP16 and BF16 (bfloat16) training. '
     'BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.'       ,    default='no'       ,    )
 parser.set_defaults(func=_lowerCAmelCase    )
 return parser







def _lowerCAmelCase     (      _lowerCAmelCase    )->     Union[str, Any]:
 __UpperCAmelCase					=   write_basic_config(args.mixed_precision       ,    args.save_location    )
 if config_file:
  print(F'accelerate configuration saved at {config_file}'    )





