



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 		UpperCAmelCase						(						_snake_case       ,unittest.TestCase				):
   SCREAMING_SNAKE_CASE__							=     CodeGenTokenizer
   SCREAMING_SNAKE_CASE__							=     CodeGenTokenizerFast
   SCREAMING_SNAKE_CASE__							=     True
   SCREAMING_SNAKE_CASE__							=     {"""add_prefix_space""": True}
   SCREAMING_SNAKE_CASE__							=     False





   def 	__lowerCAmelCase							(	self			):
         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         _lowerCAmelCase 			=  [
             "l",
             "o",
             "w",
             "e",
             "r",
             "s",
             "t",
             "i",
             "d",
             "n",
             "\u0120",
             "\u0120l",
             "\u0120n",
             "\u0120lo",
             "\u0120low",
             "er",
             "\u0120lowest",
             "\u0120newer",
             "\u0120wider",
             "<unk>",
             "<|endoftext|>",
         ]
         _lowerCAmelCase 			=  dict(zip(snake_case_       ,     range(len(snake_case_			)			)			)			)
         _lowerCAmelCase 			=  ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
         _lowerCAmelCase 			=  {"unk_token": "<unk>"}

         _lowerCAmelCase 			=  os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['''vocab_file''']			)
         _lowerCAmelCase 			=  os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['''merges_file''']			)
         with open(self.vocab_file       ,     '''w'''       ,     encoding='''utf-8'''			) as fp:
               fp.write(json.dumps(snake_case_			) + '''\n'''			)
         with open(self.merges_file       ,     '''w'''       ,     encoding='''utf-8'''			) as fp:
               fp.write('''\n'''.join(snake_case_			)			)





   def 	__lowerCAmelCase							(	self       ,     **_lowerCAmelCase			):
         kwargs.update(self.special_tokens_map			)
         return CodeGenTokenizer.from_pretrained(self.tmpdirname       ,     **snake_case_			)





   def 	__lowerCAmelCase							(	self       ,     **_lowerCAmelCase			):
         kwargs.update(self.special_tokens_map			)
         return CodeGenTokenizerFast.from_pretrained(self.tmpdirname       ,     **snake_case_			)





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  "lower newer"
         _lowerCAmelCase 			=  "lower newer"
         return input_text, output_text





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  CodeGenTokenizer(self.vocab_file       ,     self.merges_file       ,     **self.special_tokens_map			)
         _lowerCAmelCase 			=  "lower newer"
         _lowerCAmelCase 			=  ["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
         _lowerCAmelCase 			=  tokenizer.tokenize(snake_case_       ,     add_prefix_space=snake_case_			)
         self.assertListEqual(snake_case_       ,     snake_case_			)

         _lowerCAmelCase 			=  tokens + [tokenizer.unk_token]
         _lowerCAmelCase 			=  [14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_			)       ,     snake_case_			)





   def 	__lowerCAmelCase							(	self			):
         if not self.test_rust_tokenizer:
               return

         _lowerCAmelCase 			=  self.get_tokenizer()
         _lowerCAmelCase 			=  self.get_rust_tokenizer(add_prefix_space=snake_case_			)

         _lowerCAmelCase 			=  "lower newer"

         # Testing tokenization
         _lowerCAmelCase 			=  tokenizer.tokenize(snake_case_       ,     add_prefix_space=snake_case_			)
         _lowerCAmelCase 			=  rust_tokenizer.tokenize(snake_case_			)
         self.assertListEqual(snake_case_       ,     snake_case_			)

         # Testing conversion to ids without special tokens
         _lowerCAmelCase 			=  tokenizer.encode(snake_case_       ,     add_special_tokens=snake_case_       ,     add_prefix_space=snake_case_			)
         _lowerCAmelCase 			=  rust_tokenizer.encode(snake_case_       ,     add_special_tokens=snake_case_			)
         self.assertListEqual(snake_case_       ,     snake_case_			)

         # Testing conversion to ids with special tokens
         _lowerCAmelCase 			=  self.get_rust_tokenizer(add_prefix_space=snake_case_			)
         _lowerCAmelCase 			=  tokenizer.encode(snake_case_       ,     add_prefix_space=snake_case_			)
         _lowerCAmelCase 			=  rust_tokenizer.encode(snake_case_			)
         self.assertListEqual(snake_case_       ,     snake_case_			)

         # Testing the unknown token
         _lowerCAmelCase 			=  tokens + [rust_tokenizer.unk_token]
         _lowerCAmelCase 			=  [14, 15, 10, 9, 3, 2, 15, 19]
         self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(snake_case_			)       ,     snake_case_			)





   def 	__lowerCAmelCase							(	self       ,     *_lowerCAmelCase       ,     **_lowerCAmelCase			):
         # It's very difficult to mix/test pretokenization with byte-level
         # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
         pass





   def 	__lowerCAmelCase							(	self       ,     _lowerCAmelCase=15			):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
                     _lowerCAmelCase 			=  self.rust_tokenizer_class.from_pretrained(snake_case_       ,     **snake_case_			)

                     # Simple input
                     _lowerCAmelCase 			=  "This is a simple input"
                     _lowerCAmelCase 			=  ["This is a simple input 1", "This is a simple input 2"]
                     _lowerCAmelCase 			=  ("This is a simple input", "This is a pair")
                     _lowerCAmelCase 			=  [
                         ("This is a simple input 1", "This is a simple input 2"),
                         ("This is a simple pair 1", "This is a simple pair 2"),
                     ]

                     # Simple input tests
                     self.assertRaises(snake_case_       ,     tokenizer_r.encode       ,     snake_case_       ,     max_length=snake_case_       ,     padding='''max_length'''			)

                     # Simple input
                     self.assertRaises(snake_case_       ,     tokenizer_r.encode_plus       ,     snake_case_       ,     max_length=snake_case_       ,     padding='''max_length'''			)

                     # Simple input
                     self.assertRaises(
                         snake_case_       ,     tokenizer_r.batch_encode_plus       ,     snake_case_       ,     max_length=snake_case_       ,     padding='''max_length'''       ,     )

                     # Pair input
                     self.assertRaises(snake_case_       ,     tokenizer_r.encode       ,     snake_case_       ,     max_length=snake_case_       ,     padding='''max_length'''			)

                     # Pair input
                     self.assertRaises(snake_case_       ,     tokenizer_r.encode_plus       ,     snake_case_       ,     max_length=snake_case_       ,     padding='''max_length'''			)

                     # Pair input
                     self.assertRaises(
                         snake_case_       ,     tokenizer_r.batch_encode_plus       ,     snake_case_       ,     max_length=snake_case_       ,     padding='''max_length'''       ,     )





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  CodeGenTokenizer.from_pretrained(self.tmpdirname       ,     pad_token='''<pad>'''			)

         # Simple input
         _lowerCAmelCase 			=  "This is a simple input"
         _lowerCAmelCase 			=  ["This is a simple input looooooooong", "This is a simple input"]
         _lowerCAmelCase 			=  ("This is a simple input", "This is a pair")
         _lowerCAmelCase 			=  [
             ("This is a simple input loooooong", "This is a simple input"),
             ("This is a simple pair loooooong", "This is a simple pair"),
         ]

         _lowerCAmelCase 			=  tokenizer.pad_token_id

         _lowerCAmelCase 			=  tokenizer(snake_case_       ,     padding='''max_length'''       ,     max_length=30       ,     return_tensors='''np'''			)
         _lowerCAmelCase 			=  tokenizer(snake_case_       ,     padding=snake_case_       ,     truncate=snake_case_       ,     return_tensors='''np'''			)
         _lowerCAmelCase 			=  tokenizer(*snake_case_       ,     padding='''max_length'''       ,     max_length=60       ,     return_tensors='''np'''			)
         _lowerCAmelCase 			=  tokenizer(snake_case_       ,     padding=snake_case_       ,     truncate=snake_case_       ,     return_tensors='''np'''			)

         # s
         # test single string max_length padding
         self.assertEqual(out_s['''input_ids'''].shape[-1]       ,     30			)
         self.assertTrue(pad_token_id in out_s['''input_ids''']			)
         self.assertTrue(0 in out_s['''attention_mask''']			)

         # s2
         # test automatic padding
         self.assertEqual(out_sa['''input_ids'''].shape[-1]       ,     33			)
         # long slice doesn't have padding
         self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]			)
         self.assertFalse(0 in out_sa['''attention_mask'''][0]			)
         # short slice does have padding
         self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]			)
         self.assertTrue(0 in out_sa['''attention_mask'''][1]			)

         # p
         # test single pair max_length padding
         self.assertEqual(out_p['''input_ids'''].shape[-1]       ,     60			)
         self.assertTrue(pad_token_id in out_p['''input_ids''']			)
         self.assertTrue(0 in out_p['''attention_mask''']			)

         # p2
         # test automatic padding pair
         self.assertEqual(out_pa['''input_ids'''].shape[-1]       ,     52			)
         # long slice pair doesn't have padding
         self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]			)
         self.assertFalse(0 in out_pa['''attention_mask'''][0]			)
         # short slice pair does have padding
         self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]			)
         self.assertTrue(0 in out_pa['''attention_mask'''][1]			)





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  "$$$"
         _lowerCAmelCase 			=  CodeGenTokenizer.from_pretrained(self.tmpdirname       ,     bos_token=snake_case_       ,     add_bos_token=snake_case_			)

         _lowerCAmelCase 			=  "This is a simple input"
         _lowerCAmelCase 			=  ["This is a simple input 1", "This is a simple input 2"]

         _lowerCAmelCase 			=  tokenizer.bos_token_id

         _lowerCAmelCase 			=  tokenizer(snake_case_			)
         _lowerCAmelCase 			=  tokenizer(snake_case_			)

         self.assertEqual(out_s.input_ids[0]       ,     snake_case_			)
         self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

         _lowerCAmelCase 			=  tokenizer.decode(out_s.input_ids			)
         _lowerCAmelCase 			=  tokenizer.batch_decode(out_sa.input_ids			)

         self.assertEqual(decode_s.split()[0]       ,     snake_case_			)
         self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)





   @slow
   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''			)

         _lowerCAmelCase 			=  "\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
         _lowerCAmelCase 			=  "\nif len_a > len_b:      result = a\nelse:      result = b"

         _lowerCAmelCase 			=  tokenizer.encode(snake_case_			)
         _lowerCAmelCase 			=  ["^#", re.escape('''<|endoftext|>'''			), "^'''", "^\"\"\"", "\n\n\n"]
         _lowerCAmelCase 			=  tokenizer.decode(snake_case_       ,     truncate_before_pattern=snake_case_			)
         self.assertEqual(snake_case_       ,     snake_case_			)





   def 	__lowerCAmelCase							(	self			):
         pass




from __future__ import annotations

import json

import requests
from bsa import BeautifulSoup
from fake_useragent import UserAgent

UpperCAmelCase_									=							{"UserAgent": UserAgent().random}
def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	Dict				)->dict:
      _lowerCAmelCase 			=  script.contents[0]
      _lowerCAmelCase 			=  json.loads(data[data.find('''{"config"'''				) : -1]				)
      return info["entry_data"]["ProfilePage"][0]["graphql"]["user"]





class 		UpperCAmelCase						:





   def __init__(	self       ,     _lowerCAmelCase			):
         _lowerCAmelCase 			=  F'''https://www.instagram.com/{username}/'''
         _lowerCAmelCase 			=  self.get_json()





   def 	__lowerCAmelCase							(	self			):
         _lowerCAmelCase 			=  requests.get(self.url       ,     headers=_lowerCAmelCase			).text
         _lowerCAmelCase 			=  BeautifulSoup(_lowerCAmelCase       ,     '''html.parser'''			).find_all('''script'''			)
         try:
               return extract_user_profile(scripts[4]			)
         except (json.decoder.JSONDecodeError, KeyError):
               return extract_user_profile(scripts[3]			)





   def __repr__(	self			):
         return F'''{self.__class__.__name__}(\'{self.username}\')'''





   def __str__(	self			):
         return F'''{self.fullname} ({self.username}) is {self.biography}'''





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["username"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["full_name"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["biography"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["business_email"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["external_url"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["edge_followed_by"]["count"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["edge_follow"]["count"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["edge_owner_to_timeline_media"]["count"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["profile_pic_url_hd"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["is_verified"]





   @property
   def 	__lowerCAmelCase							(	self			):
         return self.user_data["is_private"]






def 							UpperCAmelCase__		(				_SCREAMING_SNAKE_CASE   :	str = "github"				)->None:
      import os

      if os.environ.get('''CI'''				):
            return  # test failing on GitHub Actions
      _lowerCAmelCase 			=  InstagramUser(_SCREAMING_SNAKE_CASE				)
      assert instagram_user.user_data
      assert isinstance(instagram_user.user_data     ,				_SCREAMING_SNAKE_CASE				)
      assert instagram_user.username == username
      if username != "github":
            return
      assert instagram_user.fullname == "GitHub"
      assert instagram_user.biography == "Built for developers."
      assert instagram_user.number_of_posts > 1_5_0
      assert instagram_user.number_of_followers > 1_2_0_0_0_0
      assert instagram_user.number_of_followings > 1_5
      assert instagram_user.email == "support@github.com"
      assert instagram_user.website == "https://github.com/readme"
      assert instagram_user.profile_picture_url.startswith('''https://instagram.'''				)
      assert instagram_user.is_verified is True
      assert instagram_user.is_private is False


if __name__ == "__main__":
    import doctest

    doctest.testmod()
    UpperCAmelCase_									=							InstagramUser("github")
    print(instagram_user)
    print(F"""{instagram_user.number_of_posts = }""")
    print(F"""{instagram_user.number_of_followers = }""")
    print(F"""{instagram_user.number_of_followings = }""")
    print(F"""{instagram_user.email = }""")
    print(F"""{instagram_user.website = }""")
    print(F"""{instagram_user.profile_picture_url = }""")
    print(F"""{instagram_user.is_verified = }""")
    print(F"""{instagram_user.is_private = }""")