





from collections import deque



class        _lowerCAmelCase    :


  '''simple docstring'''







  def __init__(      self      :					Any      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					int							):


    '''simple docstring'''





    _snake_case   :				Dict														=	process_name  # process name
    _snake_case   :				Dict														=	arrival_time  # arrival time of the process
    # completion time of finished process or last interrupted time
    _snake_case   :				Optional[Any]														=	arrival_time
    _snake_case   :				int														=	burst_time  # remaining burst time
    _snake_case   :				Optional[Any]														=	0  # total time of the process wait in ready queue
    _snake_case   :				Dict														=	0  # time from arrival time to completion time




class        _lowerCAmelCase    :


  '''simple docstring'''







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					int      ,				UpperCamelCase      :					list[int]      ,				UpperCamelCase      :					deque[Process]      ,				UpperCamelCase      :					int      ,				):


    '''simple docstring'''





    _snake_case   :				List[str]														=	number_of_queues
    # time slice of queues that round robin algorithm applied
    _snake_case   :				Optional[Any]														=	time_slices
    # unfinished process is in this ready_queue
    _snake_case   :				List[str]														=	queue
    # current time
    _snake_case   :				List[Any]														=	current_time
    # finished process is in this sequence queue
    _snake_case   :				deque[Process]														=	deque()







  def UpperCamelCase_ (      self      :					List[str]							):


    '''simple docstring'''





    _snake_case   :				Tuple														=	[]
    for i in range(len(self.finish_queue							)							):
      sequence.append(self.finish_queue[i].process_name							)
    return sequence







  def UpperCamelCase_ (      self      :					Any      ,				UpperCamelCase      :					list[Process]							):


    '''simple docstring'''





    _snake_case   :				Optional[int]														=	[]
    for i in range(len(UpperCamelCase_							)							):
      waiting_times.append(queue[i].waiting_time							)
    return waiting_times







  def UpperCamelCase_ (      self      :					List[Any]      ,				UpperCamelCase      :					list[Process]							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	[]
    for i in range(len(UpperCamelCase_							)							):
      turnaround_times.append(queue[i].turnaround_time							)
    return turnaround_times







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					list[Process]							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	[]
    for i in range(len(UpperCamelCase_							)							):
      completion_times.append(queue[i].stop_time							)
    return completion_times







  def UpperCamelCase_ (      self      :					Tuple      ,				UpperCamelCase      :					deque[Process]							):


    '''simple docstring'''





    return [q.burst_time for q in queue]







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					Process							):


    '''simple docstring'''





    process.waiting_time += self.current_time - process.stop_time
    return process.waiting_time







  def UpperCamelCase_ (      self      :					List[Any]      ,				UpperCamelCase      :					deque[Process]							):


    '''simple docstring'''





    _snake_case   :				deque[Process]														=	deque()  # sequence deque of finished process
    while len(UpperCamelCase_							) != 0:
      _snake_case   :				Dict														=	ready_queue.popleft()  # current process

      # if process's arrival time is later than current time, update current time
      if self.current_time < cp.arrival_time:
        self.current_time += cp.arrival_time

      # update waiting time of current process
      self.update_waiting_time(UpperCamelCase_							)
      # update current time
      self.current_time += cp.burst_time
      # finish the process and set the process's burst-time 0
      _snake_case   :				Any														=	0
      # set the process's turnaround time because it is finished
      _snake_case   :				Optional[Any]														=	self.current_time - cp.arrival_time
      # set the completion time
      _snake_case   :				Union[str, Any]														=	self.current_time
      # add the process to queue that has finished queue
      finished.append(UpperCamelCase_							)

    self.finish_queue.extend(UpperCamelCase_							)  # add finished process to finish queue
    # FCFS will finish all remaining processes
    return finished







  def UpperCamelCase_ (      self      :					Optional[Any]      ,				UpperCamelCase      :					deque[Process]      ,				UpperCamelCase      :					int							):


    '''simple docstring'''





    _snake_case   :				deque[Process]														=	deque()  # sequence deque of terminated process
    # just for 1 cycle and unfinished processes will go back to queue
    for _ in range(len(UpperCamelCase_							)							):
      _snake_case   :				Any														=	ready_queue.popleft()  # current process

      # if process's arrival time is later than current time, update current time
      if self.current_time < cp.arrival_time:
        self.current_time += cp.arrival_time

      # update waiting time of unfinished processes
      self.update_waiting_time(UpperCamelCase_							)
      # if the burst time of process is bigger than time-slice
      if cp.burst_time > time_slice:
        # use CPU for only time-slice
        self.current_time += time_slice
        # update remaining burst time
        cp.burst_time -= time_slice
        # update end point time
        _snake_case   :				List[str]														=	self.current_time
        # locate the process behind the queue because it is not finished
        ready_queue.append(UpperCamelCase_							)
      else:
        # use CPU for remaining burst time
        self.current_time += cp.burst_time
        # set burst time 0 because the process is finished
        _snake_case   :				Dict														=	0
        # set the finish time
        _snake_case   :				Union[str, Any]														=	self.current_time
        # update the process' turnaround time because it is finished
        _snake_case   :				str														=	self.current_time - cp.arrival_time
        # add the process to queue that has finished queue
        finished.append(UpperCamelCase_							)

    self.finish_queue.extend(UpperCamelCase_							)  # add finished process to finish queue
    # return finished processes queue and remaining processes queue
    return finished, ready_queue







  def UpperCamelCase_ (      self      :					Optional[int]							):


    '''simple docstring'''





    for i in range(self.number_of_queues - 1							):
      _snake_case   :				List[Any]														=	self.round_robin(
          self.ready_queue      ,				self.time_slices[i]							)
    #  the last queue has first_come_first_served algorithm
    self.first_come_first_served(self.ready_queue							)

    return self.finish_queue


if __name__ == "__main__":
      import doctest

      lowerCAmelCase_						   =    Process("""P1""", 0, 53)
      lowerCAmelCase_						   =    Process("""P2""", 0, 17)
      lowerCAmelCase_						   =    Process("""P3""", 0, 68)
      lowerCAmelCase_						   =    Process("""P4""", 0, 24)
      lowerCAmelCase_						   =    3
      lowerCAmelCase_						   =    [17, 25]
      lowerCAmelCase_						   =    deque([Pa, Pa, Pa, Pa])

      if len(time_slices) != number_of_queues - 1:
            raise SystemExit(0)

      doctest.testmod(extraglobs={"""queue""": deque([Pa, Pa, Pa, Pa])})

      lowerCAmelCase_						   =    Process("""P1""", 0, 53)
      lowerCAmelCase_						   =    Process("""P2""", 0, 17)
      lowerCAmelCase_						   =    Process("""P3""", 0, 68)
      lowerCAmelCase_						   =    Process("""P4""", 0, 24)
      lowerCAmelCase_						   =    3
      lowerCAmelCase_						   =    [17, 25]
      lowerCAmelCase_						   =    deque([Pa, Pa, Pa, Pa])
      lowerCAmelCase_						   =    MLFQ(number_of_queues, time_slices, queue, 0)
      lowerCAmelCase_						   =    mlfq.multi_level_feedback_queue()

      # print total waiting times of processes(P1, P2, P3, P4)
      print(
          F"""waiting time:\
        \t\t\t{MLFQ.calculate_waiting_time(mlfq, [Pa, Pa, Pa, Pa])}"""
      )
      # print completion times of processes(P1, P2, P3, P4)
      print(
          F"""completion time:\
        \t\t{MLFQ.calculate_completion_time(mlfq, [Pa, Pa, Pa, Pa])}"""
      )
      # print total turnaround times of processes(P1, P2, P3, P4)
      print(
          F"""turnaround time:\
        \t\t{MLFQ.calculate_turnaround_time(mlfq, [Pa, Pa, Pa, Pa])}"""
      )
      # print sequence of finished processes
      print(
          F"""sequence of finished processes:\
        {mlfq.calculate_sequence_of_finish_queue()}"""
      )











import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCAmelCase_						   =    logging.get_logger(__name__)

lowerCAmelCase_						   =    {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCAmelCase_						   =    {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowerCAmelCase_						   =    {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}



class        _lowerCAmelCase    (			UpperCAmelCase_							):


  '''simple docstring'''
  a_							:       Union[str, Any]      					=VOCAB_FILES_NAMES
  a_							:       List[str]      					=PRETRAINED_VOCAB_FILES_MAP
  a_							:       str      					=PRETRAINED_INIT_CONFIGURATION
  a_							:       Optional[Any]      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a_							:       List[Any]      					=RealmTokenizer







  def __init__(      self      :					List[str]      ,				UpperCamelCase      :					Optional[int]=None      ,				UpperCamelCase      :					List[Any]=None      ,				UpperCamelCase      :					Optional[Any]=True      ,				UpperCamelCase      :					Optional[Any]="[UNK]"      ,				UpperCamelCase      :					Any="[SEP]"      ,				UpperCamelCase      :					Optional[Any]="[PAD]"      ,				UpperCamelCase      :					Optional[int]="[CLS]"      ,				UpperCamelCase      :					Optional[Any]="[MASK]"      ,				UpperCamelCase      :					Dict=True      ,				UpperCamelCase      :					Optional[int]=None      ,				**UpperCamelCase      :					Union[str, Any]      ,				):


    '''simple docstring'''





    super().__init__(
        UpperCamelCase      ,				tokenizer_file=UpperCamelCase      ,				do_lower_case=UpperCamelCase      ,				unk_token=UpperCamelCase      ,				sep_token=UpperCamelCase      ,				pad_token=UpperCamelCase      ,				cls_token=UpperCamelCase      ,				mask_token=UpperCamelCase      ,				tokenize_chinese_chars=UpperCamelCase      ,				strip_accents=UpperCamelCase      ,				**UpperCamelCase      ,				)

    _snake_case   :				int														=	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
    if (
        normalizer_state.get('lowercase'      ,				UpperCamelCase							) != do_lower_case
        or normalizer_state.get('strip_accents'      ,				UpperCamelCase							) != strip_accents
        or normalizer_state.get('handle_chinese_chars'      ,				UpperCamelCase							) != tokenize_chinese_chars
    ):
      _snake_case   :				int														=	getattr(UpperCamelCase      ,				normalizer_state.pop('type'							)							)
      _snake_case   :				List[str]														=	do_lower_case
      _snake_case   :				List[Any]														=	strip_accents
      _snake_case   :				Dict														=	tokenize_chinese_chars
      _snake_case   :				Any														=	normalizer_class(**UpperCamelCase							)

    _snake_case   :				Optional[int]														=	do_lower_case







  def UpperCamelCase_ (      self      :					Dict      ,				UpperCamelCase      :					Any      ,				**UpperCamelCase      :					Union[str, Any]							):


    '''simple docstring'''





    _snake_case   :				Tuple														=	PaddingStrategy.MAX_LENGTH

    _snake_case   :				Any														=	text
    _snake_case   :				List[str]														=	kwargs.pop('text_pair'      ,				UpperCamelCase							)
    _snake_case   :				int														=	kwargs.pop('return_tensors'      ,				UpperCamelCase							)

    _snake_case   :				Optional[int]														=	{
        'input_ids': [],
        'attention_mask': [],
        'token_type_ids': [],
    }

    for idx, candidate_text in enumerate(UpperCamelCase							):
      if batch_text_pair is not None:
        _snake_case   :				List[Any]														=	batch_text_pair[idx]
      else:
        _snake_case   :				Optional[Any]														=	None

      _snake_case   :				Optional[int]														=	super().__call__(UpperCamelCase      ,				UpperCamelCase      ,				return_tensors=UpperCamelCase      ,				**UpperCamelCase							)

      _snake_case   :				str														=	encoded_candidates.get('input_ids'							)
      _snake_case   :				Tuple														=	encoded_candidates.get('attention_mask'							)
      _snake_case   :				List[str]														=	encoded_candidates.get('token_type_ids'							)

      if encoded_input_ids is not None:
        output_data["input_ids"].append(UpperCamelCase							)
      if encoded_attention_mask is not None:
        output_data["attention_mask"].append(UpperCamelCase							)
      if encoded_token_type_ids is not None:
        output_data["token_type_ids"].append(UpperCamelCase							)

    _snake_case   :				str														=	{key: item for key, item in output_data.items() if len(UpperCamelCase							) != 0}

    return BatchEncoding(UpperCamelCase      ,				tensor_type=UpperCamelCase							)







  def UpperCamelCase_ (      self      :					str      ,				UpperCamelCase      :					Optional[Any]      ,				UpperCamelCase      :					Union[str, Any]=None							):


    '''simple docstring'''





    _snake_case   :				Dict														=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					List[int]      ,				UpperCamelCase      :					Optional[List[int]] = None							):


    '''simple docstring'''





    _snake_case   :				int														=	[self.sep_token_id]
    _snake_case   :				Dict														=	[self.cls_token_id]
    if token_ids_a is None:
      return len(cls + token_ids_a + sep							) * [0]
    return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







  def UpperCamelCase_ (      self      :					Union[str, Any]      ,				UpperCamelCase      :					str      ,				UpperCamelCase      :					Optional[str] = None							):


    '''simple docstring'''





    _snake_case   :				Optional[Any]														=	self._tokenizer.model.save(UpperCamelCase      ,				name=UpperCamelCase							)
    return tuple(UpperCamelCase							)




