




from abc import ABC, abstractmethod
from typing import List, Optional




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				def __init__(   self       :  List[str]							)      ->  Union[str, Any]:
							# test for the above condition
							self.test()





				def 							__UpperCAmelCase							(   self       :  Any							)      ->  str:
							a					=    0
							a					=    False
							while not completed:
										if counter == 1:
													self.reset()
										a					=    self.advance()
										if not self.does_advance(UpperCamelCase_							):
													raise Exception(
													    "Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true."							)

										a					=    self.update(UpperCamelCase_							)
										counter += 1

										if counter > 1_00_00:
													raise Exception("update() does not fulfill the constraint."							)

							if self.remaining() != 0:
										raise Exception("Custom Constraint is not defined correctly."							)





				@abstractmethod
				def 							__UpperCAmelCase							(   self       :  List[str]							)      ->  str:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)





				@abstractmethod
				def 							__UpperCAmelCase							(   self       :  int      ,   __lowerCamelCase       :  Tuple							)      ->  Optional[Any]:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)





				@abstractmethod
				def 							__UpperCAmelCase							(   self       :  List[Any]      ,   __lowerCamelCase       :  Optional[int]							)      ->  Dict:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)





				@abstractmethod
				def 							__UpperCAmelCase							(   self       :  Optional[Any]							)      ->  Any:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)





				@abstractmethod
				def 							__UpperCAmelCase							(   self       :  List[Any]							)      ->  Union[str, Any]:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)





				@abstractmethod
				def 							__UpperCAmelCase							(   self       :  str      ,   __lowerCamelCase       :  Optional[Any]=False							)      ->  Union[str, Any]:
							raise NotImplementedError(
							    f"""{self.__class__} is an abstract class. Only classes inheriting this class can be called."""							)




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				def __init__(   self       :  List[Any]      ,   __lowerCamelCase       :  int							)      ->  List[str]:
							super(UpperCamelCase_      ,   self							).__init__()

							if not isinstance(UpperCamelCase_      ,   UpperCamelCase_							) or len(UpperCamelCase_							) == 0:
										raise ValueError(f"""`token_ids` has to be a non-empty list, but is {token_ids}."""							)
							if any((not isinstance(UpperCamelCase_      ,   UpperCamelCase_							) or token_id < 0) for token_id in token_ids							):
										raise ValueError(f"""Each list in `token_ids` has to be a list of positive integers, but is {token_ids}."""							)

							a					=    token_ids

							a					=    len(self.token_ids							)
							a					=    -1  # the index of the currently fulfilled step
							a					=    False





				def 							__UpperCAmelCase							(   self       :  Tuple							)      ->  Optional[Any]:
							if self.completed:
										return None
							return self.token_ids[self.fulfilled_idx + 1]





				def 							__UpperCAmelCase							(   self       :  List[str]      ,   __lowerCamelCase       :  Optional[Any]							)      ->  Dict:
							if not isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
										raise ValueError(f"""`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase_							)}"""							)

							if self.completed:
										return False

							return token_id == self.token_ids[self.fulfilled_idx + 1]





				def 							__UpperCAmelCase							(   self       :  Dict      ,   __lowerCamelCase       :  int							)      ->  Union[str, Any]:
							if not isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
										raise ValueError(f"""`token_id` has to be an `int`, but is {token_id} of type {type(UpperCamelCase_							)}"""							)

							a					=    False
							a					=    False
							a					=    False

							if self.does_advance(UpperCamelCase_							):
										self.fulfilled_idx += 1
										a					=    True
										if self.fulfilled_idx == (self.seqlen - 1):
													a					=    True
										a					=    completed
							else:
										# failed to make progress.
										a					=    True
										self.reset()
							return stepped, completed, reset





				def 							__UpperCAmelCase							(   self       :  List[Any]							)      ->  List[str]:
							a					=    False
							a					=    0





				def 							__UpperCAmelCase							(   self       :  Optional[int]							)      ->  Optional[Any]:
							return self.seqlen - (self.fulfilled_idx + 1)





				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  str=False							)      ->  List[Any]:
							a					=    PhrasalConstraint(self.token_ids							)

							if stateful:
										a					=    self.seqlen
										a					=    self.fulfilled_idx
										a					=    self.completed

							return new_constraint




class 					snake_case__							:
				"""simple docstring"""



				def __init__(   self       :  List[str]      ,   __lowerCamelCase       :  int      ,   __lowerCamelCase       :  Optional[Any]=True							)      ->  str:
							a					=    max([len(UpperCamelCase_							) for one in nested_token_ids]							)

							a					=    {}
							for token_ids in nested_token_ids:
										a					=    root
										for tidx, token_id in enumerate(UpperCamelCase_							):
													if token_id not in level:
																a					=    {}

													a					=    level[token_id]

							if no_subsets and self.has_subsets(UpperCamelCase_      ,   UpperCamelCase_							):
										raise ValueError(
										    "Each list in `nested_token_ids` can't be a complete subset of another list, but is"
										    f""" {nested_token_ids}."""							)

							a					=    root





				def 							__UpperCAmelCase							(   self       :  Optional[int]      ,   __lowerCamelCase       :  Optional[Any]							)      ->  Optional[int]:
							a					=    self.trie

							for current_token in current_seq:
										a					=    start[current_token]

							a					=    list(start.keys()							)

							return next_tokens





				def 							__UpperCAmelCase							(   self       :  Union[str, Any]      ,   __lowerCamelCase       :  Union[str, Any]							)      ->  Optional[int]:
							a					=    self.next_tokens(UpperCamelCase_							)

							return len(UpperCamelCase_							) == 0





				def 							__UpperCAmelCase							(   self       :  Union[str, Any]      ,   __lowerCamelCase       :  Dict							)      ->  Tuple:
							a					=    list(root.values()							)
							if len(UpperCamelCase_							) == 0:
										return 1
							else:
										return sum([self.count_leaves(UpperCamelCase_							) for nn in next_nodes]							)





				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  List[Any]      ,   __lowerCamelCase       :  List[str]							)      ->  str:
							a					=    self.count_leaves(UpperCamelCase_							)
							return len(UpperCamelCase_							) != leaf_count




class 					snake_case__							(_UpperCamelCase     ):
				"""simple docstring"""



				def __init__(   self       :  Dict      ,   __lowerCamelCase       :  Any							)      ->  int:
							super(UpperCamelCase_      ,   self							).__init__()

							if not isinstance(UpperCamelCase_      ,   UpperCamelCase_							) or len(UpperCamelCase_							) == 0:
										raise ValueError(f"""`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}."""							)
							if any(not isinstance(UpperCamelCase_      ,   UpperCamelCase_							) for token_ids in nested_token_ids							):
										raise ValueError(f"""`nested_token_ids` has to be a list of lists, but is {nested_token_ids}."""							)
							if any(
							    any((not isinstance(UpperCamelCase_      ,   UpperCamelCase_							) or token_id < 0) for token_id in token_ids							)
							    for token_ids in nested_token_ids							):
										raise ValueError(
										    f"""Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}."""							)

							a					=    DisjunctiveTrie(UpperCamelCase_							)
							a					=    nested_token_ids

							a					=    self.trie.max_height
							a					=    []
							a					=    False





				def 							__UpperCAmelCase							(   self       :  List[str]							)      ->  Dict:
							a					=    self.trie.next_tokens(self.current_seq							)

							if len(UpperCamelCase_							) == 0:
										return None
							else:
										return token_list





				def 							__UpperCAmelCase							(   self       :  Dict      ,   __lowerCamelCase       :  List[str]							)      ->  int:
							if not isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
										raise ValueError(f"""`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase_							)}"""							)

							a					=    self.trie.next_tokens(self.current_seq							)

							return token_id in next_tokens





				def 							__UpperCAmelCase							(   self       :  Optional[int]      ,   __lowerCamelCase       :  Dict							)      ->  Optional[int]:
							if not isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
										raise ValueError(f"""`token_id` is supposed to be type `int`, but is {token_id} of type {type(UpperCamelCase_							)}"""							)

							a					=    False
							a					=    False
							a					=    False

							if self.does_advance(UpperCamelCase_							):
										self.current_seq.append(UpperCamelCase_							)
										a					=    True
							else:
										a					=    True
										self.reset()

							a					=    self.trie.reached_leaf(self.current_seq							)
							a					=    completed

							return stepped, completed, reset





				def 							__UpperCAmelCase							(   self       :  Any							)      ->  Optional[Any]:
							a					=    False
							a					=    []





				def 							__UpperCAmelCase							(   self       :  List[str]							)      ->  Optional[Any]:
							if self.completed:
										# since this can be completed without reaching max height
										return 0
							else:
										return self.seqlen - len(self.current_seq							)





				def 							__UpperCAmelCase							(   self       :  Any      ,   __lowerCamelCase       :  Optional[Any]=False							)      ->  List[str]:
							a					=    DisjunctiveConstraint(self.token_ids							)

							if stateful:
										a					=    self.seqlen
										a					=    self.current_seq
										a					=    self.completed

							return new_constraint




class 					snake_case__							:
				"""simple docstring"""



				def __init__(   self       :  Optional[int]      ,   __lowerCamelCase       :  Dict							)      ->  Optional[int]:
							a					=    constraints

							# max # of steps required to fulfill a given constraint
							a					=    max([c.seqlen for c in constraints]							)
							a					=    len(UpperCamelCase_							)
							a					=    False

							self.init_state()





				def 							__UpperCAmelCase							(   self       :  str							)      ->  Union[str, Any]:
							a					=    []
							a					=    None
							a					=    [constraint.copy(stateful=UpperCamelCase_							) for constraint in self.constraints]





				def 							__UpperCAmelCase							(   self       :  List[str]							)      ->  int:
							a					=    0
							if self.inprogress_constraint:
										# extra points for having a constraint mid-fulfilled
										add += self.max_seqlen - self.inprogress_constraint.remaining()

							return (len(self.complete_constraints							) * self.max_seqlen) + add





				def 							__UpperCAmelCase							(   self       :  Tuple							)      ->  Dict:
							a					=    []
							if self.inprogress_constraint is None:
										for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
													a					=    constraint.advance()
													if isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
																token_list.append(UpperCamelCase_							)
													elif isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
																token_list.extend(UpperCamelCase_							)
							else:
										a					=    self.inprogress_constraint.advance()
										if isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
													token_list.append(UpperCamelCase_							)
										elif isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
													token_list.extend(UpperCamelCase_							)

							if len(UpperCamelCase_							) == 0:
										return None
							else:
										return token_list





				def 							__UpperCAmelCase							(   self       :  Dict      ,   __lowerCamelCase       :  Tuple							)      ->  Optional[int]:
							self.init_state()

							if token_ids is not None:
										for token in token_ids:
													# completes or steps **one** constraint
													a					=    self.add(UpperCamelCase_							)

													# the entire list of constraints are fulfilled
													if self.completed:
																break





				def 							__UpperCAmelCase							(   self       :  Dict      ,   __lowerCamelCase       :  Tuple							)      ->  Dict:
							if not isinstance(UpperCamelCase_      ,   UpperCamelCase_							):
										raise ValueError(f"""`token_id` should be an `int`, but is `{token_id}`."""							)

							a					=    False, False

							if self.completed:
										a					=    True
										a					=    False
										return complete, stepped

							if self.inprogress_constraint is not None:
										# In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
										# job, simply update the state

										a					=    self.inprogress_constraint.update(UpperCamelCase_							)
										if reset:
													# 1. If the next token breaks the progress, then we must restart.
													#     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

													#     But that doesn't mean we self.init_state(), since we only reset the state for this particular
													#     constraint, not the full list of constraints.

													self.pending_constraints.append(self.inprogress_constraint.copy(stateful=UpperCamelCase_							)							)
													a					=    None

										if complete:
													# 2. If the next token completes the constraint, move it to completed list, set
													#     inprogress to None. If there are no pending constraints either, then this full list of constraints
													#     is complete.

													self.complete_constraints.append(self.inprogress_constraint							)
													a					=    None

													if len(self.pending_constraints							) == 0:
																# we're done!
																a					=    True

							else:
										# Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
										# of constraints?

										for cidx, pending_constraint in enumerate(self.pending_constraints							):
													if pending_constraint.does_advance(UpperCamelCase_							):
																a					=    pending_constraint.update(UpperCamelCase_							)

																if not stepped:
																			raise Exception(
																			    "`constraint.update(token_id)` is not yielding incremental progress, "
																			    "even though `constraint.does_advance(token_id)` is true."							)

																if complete:
																			self.complete_constraints.append(UpperCamelCase_							)
																			a					=    None

																if not complete and stepped:
																			a					=    pending_constraint

																if complete or stepped:
																			# If we made any progress at all, then it's at least not a "pending constraint".

																			a					=    (
																			    self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
																			)

																			if len(self.pending_constraints							) == 0 and self.inprogress_constraint is None:
																						# If there's no longer any pending after this and no inprogress either, then we must be
																						# complete.

																						a					=    True

																			break  # prevent accidentally stepping through multiple constraints with just one token.

							return complete, stepped





				def 							__UpperCAmelCase							(   self       :  Optional[Any]      ,   __lowerCamelCase       :  Union[str, Any]=True							)      ->  Dict:
							a					=    ConstraintListState(self.constraints							)  # we actually never though self.constraints objects
							# throughout this process. So it's at initialization state.

							if stateful:
										a					=    [
										    constraint.copy(stateful=UpperCamelCase_							) for constraint in self.complete_constraints
										]
										if self.inprogress_constraint is not None:
													a					=    self.inprogress_constraint.copy(stateful=UpperCamelCase_							)
										a					=    [constraint.copy() for constraint in self.pending_constraints]

							return new_state












from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


__lowerCAmelCase	:       List[Any]     =	logging.get_logger(__name__)


__lowerCAmelCase	:       List[Any]     =	TypeVar('DatasetType', Dataset, IterableDataset)




def     __magic_name__       (				A	:      List[DatasetType],						A	:      Optional[List[float]] = None,						A	:      Optional[int] = None,						A	:      Optional[DatasetInfo] = None,						A	:      Optional[NamedSplit] = None,						A	:      Literal["first_exhausted", "all_exhausted"] = "first_exhausted",						):






			'''simple docstring'''







			from .arrow_dataset import Dataset
			from .iterable_dataset import IterableDataset

			if not datasets:
						raise ValueError("Unable to interleave an empty list of datasets."     )
			for i, dataset in enumerate(A     ):
						if not isinstance(A,						(Dataset, IterableDataset)     ):
									if isinstance(A,						(DatasetDict, IterableDatasetDict)     ):
												if not dataset:
															raise ValueError(
															    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} """
															    "is an empty dataset dictionary."     )
												raise ValueError(
												    F"""Dataset at position {i} has at least one split: {list(A     )}\n"""
												    F"""Please pick one to interleave with the other datasets, for example: dataset['{next(iter(A     )     )}']"""     )
									raise ValueError(
									    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(A     ).__name__}."""     )
						if i == 0:
									a					,	a					=    (
									    (Dataset, IterableDataset) if isinstance(A,						A     ) else (IterableDataset, Dataset)
									)
						elif not isinstance(A,						A     ):
									raise ValueError(
									    F"""Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."""     )
			if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
						raise ValueError(F"""{stopping_strategy} is not supported. Please enter a valid stopping_strategy."""     )
			if dataset_type is Dataset:
						return _interleave_map_style_datasets(
						    A,						A,						A,						info=A,						split=A,						stopping_strategy=A     )
			else:
						return _interleave_iterable_datasets(
						    A,						A,						A,						info=A,						split=A,						stopping_strategy=A     )






def     __magic_name__       (				A	:      List[DatasetType],						A	:      Optional[DatasetInfo] = None,						A	:      Optional[NamedSplit] = None,						A	:      int = 0,						):






			'''simple docstring'''







			if not dsets:
						raise ValueError("Unable to concatenate an empty list of datasets."     )
			for i, dataset in enumerate(A     ):
						if not isinstance(A,						(Dataset, IterableDataset)     ):
									if isinstance(A,						(DatasetDict, IterableDatasetDict)     ):
												if not dataset:
															raise ValueError(
															    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} """
															    "is an empty dataset dictionary."     )
												raise ValueError(
												    F"""Dataset at position {i} has at least one split: {list(A     )}\n"""
												    F"""Please pick one to interleave with the other datasets, for example: dataset['{next(iter(A     )     )}']"""     )
									raise ValueError(
									    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(A     ).__name__}."""     )
						if i == 0:
									a					,	a					=    (
									    (Dataset, IterableDataset) if isinstance(A,						A     ) else (IterableDataset, Dataset)
									)
						elif not isinstance(A,						A     ):
									raise ValueError(
									    F"""Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."""     )
			if dataset_type is Dataset:
						return _concatenate_map_style_datasets(A,						info=A,						split=A,						axis=A     )
			else:
						return _concatenate_iterable_datasets(A,						info=A,						split=A,						axis=A     )






