


__a     =			range(2, 20 + 1)
__a     =			[10**k for k in range(ks[-1] + 1)]
__a     =			{}
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Union[str, Any]:
   UpperCAmelCase							=     sum(a_i[j] for j in range(lowerCAmelCase_  ,						len(lowerCAmelCase_   )   )   )
   UpperCAmelCase							=     sum(a_i[j] * base[j] for j in range(min(len(lowerCAmelCase_   )  ,						lowerCAmelCase_   )   )   )

   UpperCAmelCase				,  UpperCAmelCase							=     0, 0
   UpperCAmelCase							=     n - i

   UpperCAmelCase							=     memo.get(lowerCAmelCase_   )

   if sub_memo is not None:
      UpperCAmelCase							=     sub_memo.get(lowerCAmelCase_   )

      if jumps is not None and len(lowerCAmelCase_   ) > 0:
         # find and make the largest jump without going over
         UpperCAmelCase							=     -1
         for _k in range(len(lowerCAmelCase_   ) - 1  ,						-1  ,						-1   ):
            if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
               UpperCAmelCase							=     _k
               break

         if max_jump >= 0:
            UpperCAmelCase				,  UpperCAmelCase				,  UpperCAmelCase							=     jumps[max_jump]
            # since the difference between jumps is cached, add c
            UpperCAmelCase							=     diff + c
            for j in range(min(lowerCAmelCase_  ,						len(lowerCAmelCase_   )   )   ):
               UpperCAmelCase				,  UpperCAmelCase							=     divmod(lowerCAmelCase_  ,						1_0   )
            if new_c > 0:
               add(lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )

      else:
         UpperCAmelCase							=     []
   else:
      UpperCAmelCase							=     {c: []}
      UpperCAmelCase							=     sub_memo

   if dn >= max_dn or c + diff >= base[k]:
      return diff, dn

   if k > ks[0]:
      while True:
         # keep doing smaller jumps
         UpperCAmelCase				,  UpperCAmelCase							=     next_term(lowerCAmelCase_  ,						k - 1  ,						i + dn  ,						lowerCAmelCase_   )
         diff += _diff
         dn += terms_jumped

         if dn >= max_dn or c + diff >= base[k]:
            break
   else:
      # would be too small a jump, just compute sequential terms instead
      UpperCAmelCase				,  UpperCAmelCase							=     compute(lowerCAmelCase_  ,						lowerCAmelCase_  ,						i + dn  ,						lowerCAmelCase_   )
      diff += _diff
      dn += terms_jumped

   UpperCAmelCase							=     sub_memo[c]

   # keep jumps sorted by # of terms skipped
   UpperCAmelCase							=     0
   while j < len(lowerCAmelCase_   ):
      if jumps[j][1] > dn:
         break
      j += 1

   # cache the jump for this value digitsum(b) and c
   sub_memo[c].insert(lowerCAmelCase_  ,						(diff, dn, k)   )
   return (diff, dn)
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->int:
   if i >= n:
      return 0, i
   if k > len(lowerCAmelCase_   ):
      a_i.extend([0 for _ in range(k - len(lowerCAmelCase_   )   )]   )

   # note: a_i -> b * 10^k + c
   # ds_b -> digitsum(b)
   # ds_c -> digitsum(c)
   UpperCAmelCase							=     i
   UpperCAmelCase				,  UpperCAmelCase				,  UpperCAmelCase							=     0, 0, 0
   for j in range(len(lowerCAmelCase_   )   ):
      if j >= k:
         ds_b += a_i[j]
      else:
         ds_c += a_i[j]

   while i < n:
      i += 1
      UpperCAmelCase							=     ds_c + ds_b
      diff += addend
      UpperCAmelCase							=     0
      for j in range(lowerCAmelCase_   ):
         UpperCAmelCase							=     a_i[j] + addend
         UpperCAmelCase				,  UpperCAmelCase							=     divmod(lowerCAmelCase_  ,						1_0   )

         ds_c += a_i[j]

      if addend > 0:
         break

   if addend > 0:
      add(lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )
   return diff, i - start_i
def   _UpperCamelCase				(						lowerCAmelCase_  ,						lowerCAmelCase_  ,						lowerCAmelCase_   )						->Optional[int]:
   for j in range(lowerCAmelCase_  ,						len(lowerCAmelCase_   )   ):
      UpperCAmelCase							=     digits[j] + addend
      if s >= 1_0:
         UpperCAmelCase				,  UpperCAmelCase							=     divmod(lowerCAmelCase_  ,						1_0   )
         UpperCAmelCase							=     addend // 1_0 + quotient
      else:
         UpperCAmelCase							=     s
         UpperCAmelCase							=     addend // 1_0

      if addend == 0:
         break

   while addend > 0:
      UpperCAmelCase				,  UpperCAmelCase							=     divmod(lowerCAmelCase_  ,						1_0   )
      digits.append(lowerCAmelCase_   )
def   _UpperCamelCase				(						lowerCAmelCase_ = 1_0**1_5   )						->int:
   UpperCAmelCase							=     [1]
   UpperCAmelCase							=     1
   UpperCAmelCase							=     0
   while True:
      UpperCAmelCase				,  UpperCAmelCase							=     next_term(lowerCAmelCase_  ,						2_0  ,						i + dn  ,						lowerCAmelCase_   )
      dn += terms_jumped
      if dn == n - i:
         break

   UpperCAmelCase							=     0
   for j in range(len(lowerCAmelCase_   )   ):
      a_n += digits[j] * 1_0**j
   return a_n


if __name__ == "__main__":
 print(F"""{solution() = }""")







import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser
def   _UpperCamelCase				(						lowerCAmelCase_   )						->int:
   UpperCAmelCase							=     {}
   UpperCAmelCase							=     tokenizer(example["""content"""]  ,						truncation=lowerCAmelCase_   )["""input_ids"""]
   UpperCAmelCase							=     len(example["""content"""]   ) / len(output["""input_ids"""]   )
   return output


__a     =			HfArgumentParser(PretokenizationArguments)
__a     =			parser.parse_args()
if args.num_workers is None:
 __a     =			multiprocessing.cpu_count()
__a     =			AutoTokenizer.from_pretrained(args.tokenizer_dir)

__a     =			time.time()
__a     =			load_dataset(args.dataset_name, split="""train""")
print(F"""Dataset loaded in {time.time()-t_start:.2f}s""")

__a     =			time.time()
__a     =			ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(F"""Dataset tokenized in {time.time()-t_start:.2f}s""")

__a     =			time.time()
ds.push_to_hub(args.tokenized_data_repo)







print(F"""Data pushed to the hub in {time.time()-t_start:.2f}s""")



