



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 						UpperCAmelCase_					( UpperCamelCase_     ,      unittest.TestCase):
       lowerCamelCase__         =	CodeGenTokenizer
       lowerCamelCase__         =	CodeGenTokenizerFast
       lowerCamelCase__         =	True
       lowerCamelCase__         =	{"""add_prefix_space""": True}
       lowerCamelCase__         =	False







       def      snake_case__   (     self):

              '''simple docstring'''

              super().setUp()

              # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
              _lowerCAmelCase  :							Union[str, Any]				     =					[
                  """l""",
                  """o""",
                  """w""",
                  """e""",
                  """r""",
                  """s""",
                  """t""",
                  """i""",
                  """d""",
                  """n""",
                  """\u0120""",
                  """\u0120l""",
                  """\u0120n""",
                  """\u0120lo""",
                  """\u0120low""",
                  """er""",
                  """\u0120lowest""",
                  """\u0120newer""",
                  """\u0120wider""",
                  """<unk>""",
                  """<|endoftext|>""",
              ]
              _lowerCAmelCase  :							Any				     =					dict(zip(_a,    range(len(_a))))
              _lowerCAmelCase  :							Optional[int]				     =					["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
              _lowerCAmelCase  :							Any				     =					{"""unk_token""": """<unk>"""}

              _lowerCAmelCase  :							Optional[Any]				     =					os.path.join(self.tmpdirname,    VOCAB_FILES_NAMES["vocab_file"])
              _lowerCAmelCase  :							Any				     =					os.path.join(self.tmpdirname,    VOCAB_FILES_NAMES["merges_file"])
              with open(self.vocab_file,    "w",    encoding="utf-8") as fp:
                     fp.write(json.dumps(_a) + "\n")
              with open(self.merges_file,    "w",    encoding="utf-8") as fp:
                     fp.write("\n".join(_a))







       def      snake_case__   (     self,    **__a):

              '''simple docstring'''

              kwargs.update(self.special_tokens_map)
              return CodeGenTokenizer.from_pretrained(self.tmpdirname,    **_a)







       def      snake_case__   (     self,    **__a):

              '''simple docstring'''

              kwargs.update(self.special_tokens_map)
              return CodeGenTokenizerFast.from_pretrained(self.tmpdirname,    **_a)







       def      snake_case__   (     self,    __a):

              '''simple docstring'''

              _lowerCAmelCase  :							List[str]				     =					"""lower newer"""
              _lowerCAmelCase  :							Optional[int]				     =					"""lower newer"""
              return input_text, output_text







       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							List[Any]				     =					CodeGenTokenizer(self.vocab_file,    self.merges_file,    **self.special_tokens_map)
              _lowerCAmelCase  :							Tuple				     =					"""lower newer"""
              _lowerCAmelCase  :							List[Any]				     =					["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
              _lowerCAmelCase  :							Optional[int]				     =					tokenizer.tokenize(_a,    add_prefix_space=_a)
              self.assertListEqual(_a,    _a)

              _lowerCAmelCase  :							Tuple				     =					tokens + [tokenizer.unk_token]
              _lowerCAmelCase  :							Optional[int]				     =					[14, 15, 10, 9, 3, 2, 15, 19]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(_a),    _a)







       def      snake_case__   (     self):

              '''simple docstring'''

              if not self.test_rust_tokenizer:
                     return

              _lowerCAmelCase  :							int				     =					self.get_tokenizer()
              _lowerCAmelCase  :							str				     =					self.get_rust_tokenizer(add_prefix_space=_a)

              _lowerCAmelCase  :							str				     =					"""lower newer"""

              # Testing tokenization
              _lowerCAmelCase  :							Optional[int]				     =					tokenizer.tokenize(_a,    add_prefix_space=_a)
              _lowerCAmelCase  :							int				     =					rust_tokenizer.tokenize(_a)
              self.assertListEqual(_a,    _a)

              # Testing conversion to ids without special tokens
              _lowerCAmelCase  :							int				     =					tokenizer.encode(_a,    add_special_tokens=_a,    add_prefix_space=_a)
              _lowerCAmelCase  :							Union[str, Any]				     =					rust_tokenizer.encode(_a,    add_special_tokens=_a)
              self.assertListEqual(_a,    _a)

              # Testing conversion to ids with special tokens
              _lowerCAmelCase  :							Tuple				     =					self.get_rust_tokenizer(add_prefix_space=_a)
              _lowerCAmelCase  :							List[str]				     =					tokenizer.encode(_a,    add_prefix_space=_a)
              _lowerCAmelCase  :							str				     =					rust_tokenizer.encode(_a)
              self.assertListEqual(_a,    _a)

              # Testing the unknown token
              _lowerCAmelCase  :							List[str]				     =					tokens + [rust_tokenizer.unk_token]
              _lowerCAmelCase  :							Tuple				     =					[14, 15, 10, 9, 3, 2, 15, 19]
              self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_a),    _a)







       def      snake_case__   (     self,    *__a,    **__a):

              '''simple docstring'''

              pass







       def      snake_case__   (     self,    __a=15):

              '''simple docstring'''

              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                            _lowerCAmelCase  :							Any				     =					self.rust_tokenizer_class.from_pretrained(_a,    **_a)

                            # Simple input
                            _lowerCAmelCase  :							int				     =					"""This is a simple input"""
                            _lowerCAmelCase  :							Optional[Any]				     =					["""This is a simple input 1""", """This is a simple input 2"""]
                            _lowerCAmelCase  :							Optional[Any]				     =					("""This is a simple input""", """This is a pair""")
                            _lowerCAmelCase  :							int				     =					[
                                ("""This is a simple input 1""", """This is a simple input 2"""),
                                ("""This is a simple pair 1""", """This is a simple pair 2"""),
                            ]

                            # Simple input tests
                            self.assertRaises(_a,    tokenizer_r.encode,    _a,    max_length=_a,    padding="max_length")

                            # Simple input
                            self.assertRaises(_a,    tokenizer_r.encode_plus,    _a,    max_length=_a,    padding="max_length")

                            # Simple input
                            self.assertRaises(
                                _a,    tokenizer_r.batch_encode_plus,    _a,    max_length=_a,    padding="max_length",    )

                            # Pair input
                            self.assertRaises(_a,    tokenizer_r.encode,    _a,    max_length=_a,    padding="max_length")

                            # Pair input
                            self.assertRaises(_a,    tokenizer_r.encode_plus,    _a,    max_length=_a,    padding="max_length")

                            # Pair input
                            self.assertRaises(
                                _a,    tokenizer_r.batch_encode_plus,    _a,    max_length=_a,    padding="max_length",    )







       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							List[str]				     =					CodeGenTokenizer.from_pretrained(self.tmpdirname,    pad_token="<pad>")

              # Simple input
              _lowerCAmelCase  :							Dict				     =					"""This is a simple input"""
              _lowerCAmelCase  :							Optional[Any]				     =					["""This is a simple input looooooooong""", """This is a simple input"""]
              _lowerCAmelCase  :							List[str]				     =					("""This is a simple input""", """This is a pair""")
              _lowerCAmelCase  :							List[str]				     =					[
                  ("""This is a simple input loooooong""", """This is a simple input"""),
                  ("""This is a simple pair loooooong""", """This is a simple pair"""),
              ]

              _lowerCAmelCase  :							List[Any]				     =					tokenizer.pad_token_id

              _lowerCAmelCase  :							List[Any]				     =					tokenizer(_a,    padding="max_length",    max_length=30,    return_tensors="np")
              _lowerCAmelCase  :							List[Any]				     =					tokenizer(_a,    padding=_a,    truncate=_a,    return_tensors="np")
              _lowerCAmelCase  :							List[str]				     =					tokenizer(*_a,    padding="max_length",    max_length=60,    return_tensors="np")
              _lowerCAmelCase  :							Tuple				     =					tokenizer(_a,    padding=_a,    truncate=_a,    return_tensors="np")

              # s
              # test single string max_length padding
              self.assertEqual(out_s["input_ids"].shape[-1],    30)
              self.assertTrue(pad_token_id in out_s["input_ids"])
              self.assertTrue(0 in out_s["attention_mask"])

              # s2
              # test automatic padding
              self.assertEqual(out_sa["input_ids"].shape[-1],    33)
              # long slice doesn't have padding
              self.assertFalse(pad_token_id in out_sa["input_ids"][0])
              self.assertFalse(0 in out_sa["attention_mask"][0])
              # short slice does have padding
              self.assertTrue(pad_token_id in out_sa["input_ids"][1])
              self.assertTrue(0 in out_sa["attention_mask"][1])

              # p
              # test single pair max_length padding
              self.assertEqual(out_p["input_ids"].shape[-1],    60)
              self.assertTrue(pad_token_id in out_p["input_ids"])
              self.assertTrue(0 in out_p["attention_mask"])

              # p2
              # test automatic padding pair
              self.assertEqual(out_pa["input_ids"].shape[-1],    52)
              # long slice pair doesn't have padding
              self.assertFalse(pad_token_id in out_pa["input_ids"][0])
              self.assertFalse(0 in out_pa["attention_mask"][0])
              # short slice pair does have padding
              self.assertTrue(pad_token_id in out_pa["input_ids"][1])
              self.assertTrue(0 in out_pa["attention_mask"][1])







       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							Optional[int]				     =					"""$$$"""
              _lowerCAmelCase  :							Dict				     =					CodeGenTokenizer.from_pretrained(self.tmpdirname,    bos_token=_a,    add_bos_token=_a)

              _lowerCAmelCase  :							Union[str, Any]				     =					"""This is a simple input"""
              _lowerCAmelCase  :							Optional[int]				     =					["""This is a simple input 1""", """This is a simple input 2"""]

              _lowerCAmelCase  :							Optional[Any]				     =					tokenizer.bos_token_id

              _lowerCAmelCase  :							List[Any]				     =					tokenizer(_a)
              _lowerCAmelCase  :							Tuple				     =					tokenizer(_a)

              self.assertEqual(out_s.input_ids[0],    _a)
              self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

              _lowerCAmelCase  :							Union[str, Any]				     =					tokenizer.decode(out_s.input_ids)
              _lowerCAmelCase  :							Dict				     =					tokenizer.batch_decode(out_sa.input_ids)

              self.assertEqual(decode_s.split()[0],    _a)
              self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))







       @slow
       def      snake_case__   (     self):

              '''simple docstring'''

              _lowerCAmelCase  :							Union[str, Any]				     =					CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono")

              _lowerCAmelCase  :							Tuple				     =					"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
              _lowerCAmelCase  :							Tuple				     =					"""\nif len_a > len_b:      result = a\nelse:      result = b"""

              _lowerCAmelCase  :							List[Any]				     =					tokenizer.encode(_a)
              _lowerCAmelCase  :							Dict				     =					["""^#""", re.escape("<|endoftext|>"), """^'''""", """^\"\"\"""", """\n\n\n"""]
              _lowerCAmelCase  :							int				     =					tokenizer.decode(_a,    truncate_before_pattern=_a)
              self.assertEqual(_a,    _a)







       def      snake_case__   (     self):

              '''simple docstring'''

              pass





import requests
from bsa import BeautifulSoup






def        A       (      _lowerCamelCase = "https://www.worldometers.info/coronavirus"						):





							'''simple docstring'''




							_lowerCAmelCase  :							str				     =					BeautifulSoup(requests.get(_lowerCamelCase						).text       ,				"html.parser"						)
							_lowerCAmelCase  :							str				     =					soup.findAll("h1"						)
							_lowerCAmelCase  :							Optional[int]				     =					soup.findAll("div"       ,				{"class": "maincounter-number"}						)
							keys += soup.findAll("span"       ,				{"class": "panel-title"}						)
							values += soup.findAll("div"       ,				{"class": "number-table-main"}						)
							return {key.text.strip(): value.text.strip() for key, value in zip(_lowerCamelCase       ,				_lowerCamelCase						)}


if __name__ == "__main__":
						print("\033[1m" + "COVID-19 Status of the World" + "\033[0m\n")
						for key, value in world_covidaa_stats().items():
												print(f'''{key}\n{value}\n''')
