



import argparse
import json
import os
from pathlib import Path

import requests
import torch

from transformers import JukeboxConfig, JukeboxModel
from transformers.utils import logging


logging.set_verbosity_info()
a_			=					logging.get_logger(__name__)


a_			=					"https://openaipublic.azureedge.net/jukebox/models/"
a_			=					{
    "jukebox-1b-lyrics": [
        "5b/vqvae.pth.tar",
        "5b/prior_level_0.pth.tar",
        "5b/prior_level_1.pth.tar",
        "1b_lyrics/prior_level_2.pth.tar",
    ],
    "jukebox-5b-lyrics": [
        "5b/vqvae.pth.tar",
        "5b/prior_level_0.pth.tar",
        "5b/prior_level_1.pth.tar",
        "5b_lyrics/prior_level_2.pth.tar",
    ],
}



def 		a__			(		_UpperCamelCase			: Tuple							):
   if key.endswith('''.model.1.bias'''							) and len(key.split('''.'''							)							) > 10:
      __lowerCamelCase													=    key.replace('''.model.1.bias''' ,'''.conv1d_1.bias'''							)
   elif key.endswith('''.model.1.weight'''							) and len(key.split('''.'''							)							) > 10:
      __lowerCamelCase													=    key.replace('''.model.1.weight''' ,'''.conv1d_1.weight'''							)
   elif key.endswith('''.model.3.bias'''							) and len(key.split('''.'''							)							) > 10:
      __lowerCamelCase													=    key.replace('''.model.3.bias''' ,'''.conv1d_2.bias'''							)
   elif key.endswith('''.model.3.weight'''							) and len(key.split('''.'''							)							) > 10:
      __lowerCamelCase													=    key.replace('''.model.3.weight''' ,'''.conv1d_2.weight'''							)

   if "conditioner_blocks.0." in key:
      __lowerCamelCase													=    key.replace('''conditioner_blocks.0''' ,'''conditioner_blocks'''							)

   if "prime_prior" in key:
      __lowerCamelCase													=    key.replace('''prime_prior''' ,'''encoder'''							)

   if ".emb." in key and "total" not in key and "absolute" not in key and "relative" not in key:
      __lowerCamelCase													=    key.replace('''.emb.''' ,'''.'''							)

   if key.endswith('''k'''							):  # replace vqvae.X.k with vqvae.X.codebook
      return key.replace('''.k''' ,'''.codebook'''							)
   if "y_emb." in key:
      return key.replace('''y_emb.''' ,'''metadata_embedding.'''							)

   if "x_emb.emb." in key:
      __lowerCamelCase													=    key.replace('''0.x_emb.emb''' ,'''embed_tokens'''							)

   if "prime_state_ln" in key:
      return key.replace('''prime_state_ln''' ,'''encoder.final_layer_norm'''							)
   if ".ln" in key:
      return key.replace('''.ln''' ,'''.layer_norm'''							)
   if "_ln" in key:
      return key.replace('''_ln''' ,'''_layer_norm'''							)

   if "prime_state_proj" in key:
      return key.replace('''prime_state_proj''' ,'''encoder.proj_in'''							)
   if "prime_x_out" in key:
      return key.replace('''prime_x_out''' ,'''encoder.lm_head'''							)
   if "prior.x_out" in key:
      return key.replace('''x_out''' ,'''fc_proj_out'''							)
   if "x_emb" in key:
      return key.replace('''x_emb''' ,'''embed_tokens'''							)

   return key



def 		a__			(		_UpperCamelCase			: str ,_UpperCamelCase			: Tuple ,_UpperCamelCase			: Union[str, Any] ,_UpperCamelCase			: List[Any]							):
   __lowerCamelCase													=    {}
   import re

   __lowerCamelCase													=    re.compile(R'''encoders.(\d*).level_blocks.(\d*).model.(\d*).(\d).(bias|weight)'''							)
   __lowerCamelCase													=    re.compile(
       R'''encoders.(\d*).level_blocks.(\d*).model.(\d*).(\d).model.(\d*).model.(\d*).(bias|weight)'''							)
   __lowerCamelCase													=    re.compile(R'''encoders.(\d*).level_blocks.(\d*).model.(\d*).(bias|weight)'''							)

   __lowerCamelCase													=    re.compile(R'''decoders.(\d*).level_blocks.(\d*).model.(\d*).(\d).(bias|weight)'''							)
   __lowerCamelCase													=    re.compile(
       R'''decoders.(\d*).level_blocks.(\d*).model.(\d*).(\d).model.(\d*).model.(\d*).(bias|weight)'''							)
   __lowerCamelCase													=    re.compile(R'''decoders.(\d*).level_blocks.(\d*).model.(\d*).(bias|weight)'''							)

   __lowerCamelCase													=    re.compile(R'''conditioner_blocks.(\d*).cond.model.(\d*).(\d).(bias|weight)'''							)
   __lowerCamelCase													=    re.compile(
       R'''conditioner_blocks.(\d*).cond.model.(\d*).(\d).model.(\d*).model.(\d*).(bias|weight)'''							)
   __lowerCamelCase													=    re.compile(R'''conditioner_blocks.(\d*).cond.model.(\d*).(bias|weight)'''							)

   for original_key, value in state_dict.items():
      # rename vqvae.encoder keys
      if re_encoder_block_conv_in.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_encoder_block_conv_in.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    int(groups[2]							) * 2 + int(groups[3]							)
         __lowerCamelCase													=    F"""encoders.{groups[0]}.level_blocks.{groups[1]}.downsample_block.{block_index}.{groups[-1]}"""
         __lowerCamelCase													=    re_encoder_block_conv_in.sub(_UpperCamelCase ,_UpperCamelCase							)

      elif re_encoder_block_resnet.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_encoder_block_resnet.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    int(groups[2]							) * 2 + int(groups[3]							)
         __lowerCamelCase													=    {'''1''': 1, '''3''': 2}[groups[-2]]
         __lowerCamelCase													=    F"""encoders.{groups[0]}.level_blocks.{groups[1]}.downsample_block.{block_index}."""
         __lowerCamelCase													=    F"""resnet_block.{groups[-3]}.conv1d_{conv_index}.{groups[-1]}"""
         __lowerCamelCase													=    prefix + resnet_block
         __lowerCamelCase													=    re_encoder_block_resnet.sub(_UpperCamelCase ,_UpperCamelCase							)

      elif re_encoder_block_proj_out.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_encoder_block_proj_out.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    F"""encoders.{groups[0]}.level_blocks.{groups[1]}.proj_out.{groups[-1]}"""
         __lowerCamelCase													=    re_encoder_block_proj_out.sub(_UpperCamelCase ,_UpperCamelCase							)

      # rename vqvae.decoder keys
      elif re_decoder_block_conv_out.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_decoder_block_conv_out.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    int(groups[2]							) * 2 + int(groups[3]							) - 2
         __lowerCamelCase													=    F"""decoders.{groups[0]}.level_blocks.{groups[1]}.upsample_block.{block_index}.{groups[-1]}"""
         __lowerCamelCase													=    re_decoder_block_conv_out.sub(_UpperCamelCase ,_UpperCamelCase							)

      elif re_decoder_block_resnet.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_decoder_block_resnet.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    int(groups[2]							) * 2 + int(groups[3]							) - 2
         __lowerCamelCase													=    {'''1''': 1, '''3''': 2}[groups[-2]]
         __lowerCamelCase													=    F"""decoders.{groups[0]}.level_blocks.{groups[1]}.upsample_block.{block_index}."""
         __lowerCamelCase													=    F"""resnet_block.{groups[-3]}.conv1d_{conv_index}.{groups[-1]}"""
         __lowerCamelCase													=    prefix + resnet_block
         __lowerCamelCase													=    re_decoder_block_resnet.sub(_UpperCamelCase ,_UpperCamelCase							)

      elif re_decoder_block_proj_in.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_decoder_block_proj_in.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    F"""decoders.{groups[0]}.level_blocks.{groups[1]}.proj_in.{groups[-1]}"""
         __lowerCamelCase													=    re_decoder_block_proj_in.sub(_UpperCamelCase ,_UpperCamelCase							)

      # rename prior cond.model to upsampler.upsample_block and resnet
      elif re_prior_cond_conv_out.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_prior_cond_conv_out.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    int(groups[1]							) * 2 + int(groups[2]							) - 2
         __lowerCamelCase													=    F"""conditioner_blocks.upsampler.upsample_block.{block_index}.{groups[-1]}"""
         __lowerCamelCase													=    re_prior_cond_conv_out.sub(_UpperCamelCase ,_UpperCamelCase							)

      elif re_prior_cond_resnet.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_prior_cond_resnet.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    int(groups[1]							) * 2 + int(groups[2]							) - 2
         __lowerCamelCase													=    {'''1''': 1, '''3''': 2}[groups[-2]]
         __lowerCamelCase													=    F"""conditioner_blocks.upsampler.upsample_block.{block_index}."""
         __lowerCamelCase													=    F"""resnet_block.{groups[-3]}.conv1d_{conv_index}.{groups[-1]}"""
         __lowerCamelCase													=    prefix + resnet_block
         __lowerCamelCase													=    re_prior_cond_resnet.sub(_UpperCamelCase ,_UpperCamelCase							)

      elif re_prior_cond_proj_in.fullmatch(_UpperCamelCase							):
         __lowerCamelCase													=    re_prior_cond_proj_in.match(_UpperCamelCase							)
         __lowerCamelCase													=    regex_match.groups()
         __lowerCamelCase													=    F"""conditioner_blocks.upsampler.proj_in.{groups[-1]}"""
         __lowerCamelCase													=    re_prior_cond_proj_in.sub(_UpperCamelCase ,_UpperCamelCase							)

      # keep original key
      else:
         __lowerCamelCase													=    original_key

      __lowerCamelCase													=    replace_key(_UpperCamelCase							)

      if F"""{key_prefix}.{key}""" not in model_state_dict or key is None:
         print(F"""failed converting {original_key} to {key}, does not match"""							)

      # handle missmatched shape
      elif value.shape != model_state_dict[F"""{key_prefix}.{key}"""].shape:
         __lowerCamelCase													=    model_state_dict[F"""{key_prefix}.{key}"""]
         print(F"""{original_key}-> {key} : \nshape {val.shape} and { value.shape}, do not match"""							)
         __lowerCamelCase													=    original_key

      __lowerCamelCase													=    original_key
      __lowerCamelCase													=    value

   return new_dict



@torch.no_grad()
def 		a__			(		_UpperCamelCase			: List[Any]=None ,_UpperCamelCase			: int=None							):
   for file in MODEL_MAPPING[model_name]:
      if not os.path.isfile(F"""{pytorch_dump_folder_path}/{file.split("/"							)[-1]}"""							):
         __lowerCamelCase													=    requests.get(F"""{PREFIX}{file}""" ,allow_redirects=_UpperCamelCase							)
         os.makedirs(F"""{pytorch_dump_folder_path}/""" ,exist_ok=_UpperCamelCase							)
         open(F"""{pytorch_dump_folder_path}/{file.split("/"							)[-1]}""" ,'''wb'''							).write(r.content							)

   __lowerCamelCase													=    MODEL_MAPPING[model_name.split('''/'''							)[-1]]

   __lowerCamelCase													=    JukeboxConfig.from_pretrained(_UpperCamelCase							)
   __lowerCamelCase													=    JukeboxModel(_UpperCamelCase							)

   __lowerCamelCase													=    []
   __lowerCamelCase													=    {}
   for i, dict_name in enumerate(_UpperCamelCase							):
      __lowerCamelCase													=    torch.load(F"""{pytorch_dump_folder_path}/{dict_name.split("/"							)[-1]}"""							)['''model''']

      __lowerCamelCase													=    {}
      for k in old_dic.keys():
         if k.endswith('''.b'''							):
            __lowerCamelCase													=    old_dic[k]
         elif k.endswith('''.w'''							):
            __lowerCamelCase													=    old_dic[k]
         elif "level_2" not in dict_name and "cond.model." in k:
            __lowerCamelCase													=    old_dic[k]
         else:
            __lowerCamelCase													=    old_dic[k]

      __lowerCamelCase													=    '''vqvae''' if i == 0 else F"""priors.{3 - i}"""
      __lowerCamelCase													=    fix_jukebox_keys(_UpperCamelCase ,model.state_dict() ,_UpperCamelCase ,_UpperCamelCase							)
      weight_dict.append(_UpperCamelCase							)

   __lowerCamelCase													=    weight_dict.pop(0							)
   model.vqvae.load_state_dict(_UpperCamelCase							)
   for i in range(len(_UpperCamelCase							)							):
      model.priors[i].load_state_dict(weight_dict[2 - i]							)

   Path(_UpperCamelCase							).mkdir(exist_ok=_UpperCamelCase							)
   with open(F"""{pytorch_dump_folder_path}/mapping.json""" ,'''w'''							) as txtfile:
      json.dump(_UpperCamelCase ,_UpperCamelCase							)

   print(F"""Saving model {model_name} to {pytorch_dump_folder_path}"""							)
   model.save_pretrained(_UpperCamelCase							)

   return weight_dict


if __name__ == "__main__":
  a_			=					argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      """--model_name""",
      default="""jukebox-5b-lyrics""",
      type=str,
      help="""Name of the model you'd like to convert.""",
  )
  parser.add_argument(
      """--pytorch_dump_folder_path""",
      default="""jukebox-5b-lyrics-converted""",
      type=str,
      help="""Path to the output PyTorch model directory.""",
  )
  a_			=					parser.parse_args()
  convert_openai_checkpoint(args.model_name, args.pytorch_dump_folder_path)











import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


a_			=					get_tests_dir("""fixtures/test_sentencepiece.model""")



@require_sentencepiece
class  __lowerCAmelCase     (	lowerCAmelCase__  ,	unittest.TestCase   ):
 lowerCAmelCase__       					=  XLMProphetNetTokenizer
 lowerCAmelCase__       					=  False
 lowerCAmelCase__       					=  True




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    super().setUp()

    # We have a SentencePiece fixture for testing
    __lowerCamelCase													=    XLMProphetNetTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)
    tokenizer.save_pretrained(self.tmpdirname				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''[PAD]'''
    __lowerCamelCase													=    0

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase				)	,				__UpperCAmelCase				)
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase				)	,				__UpperCAmelCase				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    list(self.get_tokenizer().get_vocab().keys()				)

    self.assertEqual(vocab_keys[0]	,				'''[PAD]'''				)
    self.assertEqual(vocab_keys[1]	,				'''[CLS]'''				)
    self.assertEqual(vocab_keys[-1]	,				'''j'''				)
    self.assertEqual(len(__UpperCAmelCase				)	,				1012				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    self.assertEqual(self.get_tokenizer().vocab_size	,				1012				)




 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    XLMProphetNetTokenizer(__UpperCAmelCase	,				keep_accents=__UpperCAmelCase				)

    __lowerCamelCase													=    tokenizer.tokenize('''This is a test'''				)
    self.assertListEqual(__UpperCAmelCase	,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)	,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]	,				)

    __lowerCamelCase													=    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''9''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''é''',
            '''.''',
        ]	,				)
    __lowerCamelCase													=    tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            value + tokenizer.fairseq_offset
            for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
        ]	,				)

    __lowerCamelCase													=    tokenizer.convert_ids_to_tokens(__UpperCAmelCase				)
    self.assertListEqual(
        __UpperCAmelCase	,				[
            SPIECE_UNDERLINE + '''I''',
            SPIECE_UNDERLINE + '''was''',
            SPIECE_UNDERLINE + '''b''',
            '''or''',
            '''n''',
            SPIECE_UNDERLINE + '''in''',
            SPIECE_UNDERLINE + '''''',
            '''[UNK]''',
            '''2''',
            '''0''',
            '''0''',
            '''0''',
            ''',''',
            SPIECE_UNDERLINE + '''and''',
            SPIECE_UNDERLINE + '''this''',
            SPIECE_UNDERLINE + '''is''',
            SPIECE_UNDERLINE + '''f''',
            '''al''',
            '''s''',
            '''[UNK]''',
            '''.''',
        ]	,				)




 @cached_property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    __lowerCamelCase													=    '''Hello World!'''
    __lowerCamelCase													=    [35389, 6672, 49, 2]
    self.assertListEqual(__UpperCAmelCase	,				self.big_tokenizer.encode(__UpperCAmelCase				)				)




 @slow
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    # fmt: off
    __lowerCamelCase													=    {'''input_ids''': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
    # fmt: on

    self.tokenizer_integration_test_util(
        expected_encoding=__UpperCAmelCase	,				model_name='''microsoft/xprophetnet-large-wiki100-cased'''	,				revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''	,				)






