

import logging
import os
import threading
import time


try:
							import warnings
except ImportError:
							lowerCAmelCase__								=		None

try:
							import msvcrt
except ImportError:
							lowerCAmelCase__								=		None

try:
							import fcntl
except ImportError:
							lowerCAmelCase__								=		None


# Backward compatibility
# ------------------------------------------------
try:
							TimeoutError
except NameError:
							lowerCAmelCase__								=		OSError


# Data
# ------------------------------------------------
lowerCAmelCase__								=		[
    """Timeout""",
    """BaseFileLock""",
    """WindowsFileLock""",
    """UnixFileLock""",
    """SoftFileLock""",
    """FileLock""",
]

lowerCAmelCase__								=		"""3.0.12"""


lowerCAmelCase__								=		None






def 					lowerCamelCase_     (							)						->				int:




	'''simple docstring'''




	global _logger
	_UpperCamelCase   :		Dict			=						_logger or logging.getLogger(__name__	)
	return _logger







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		def __init__(				self	,       __snake_case):
			_UpperCamelCase   :		Dict			=						lock_file
			return None






		def __str__(				self):
			_UpperCamelCase   :		Tuple			=						f'''The file lock \'{self.lock_file}\' could not be acquired.'''
			return temp







class 	lowercase    :


		"""simple docstring"""




		def __init__(				self	,       __snake_case):
			_UpperCamelCase   :		Union[str, Any]			=						lock
			return None


		def __enter__(				self):
			return self.lock






		def __exit__(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			self.lock.release()
			return None







class 	lowercase    :


		"""simple docstring"""




		def __init__(				self	,       __snake_case	,       __snake_case=-1	,       __snake_case=None):
			_UpperCamelCase   :		Optional[int]			=						max_filename_length if max_filename_length is not None else 2_55
			# Hash the filename if it's too long
			_UpperCamelCase   :		Dict			=						self.hash_filename_if_too_long(__snake_case	,       __snake_case)
			# The path to the lock file.
			_UpperCamelCase   :		Tuple			=						lock_file

			# The file descriptor for the *_lock_file* as it is returned by the
			# os.open() function.
			# This file lock is only NOT None, if the object currently holds the
			# lock.
			_UpperCamelCase   :		Dict			=						None

			# The default timeout value.
			_UpperCamelCase   :		str			=						timeout

			# We use this lock primarily for the lock counter.
			_UpperCamelCase   :		Any			=						threading.Lock()

			# The lock counter is used for implementing the nested locking
			# mechanism. Whenever the lock is acquired, the counter is increased and
			# the lock is only released, when this value is 0 again.
			_UpperCamelCase   :		Union[str, Any]			=						0
			return None


		@property
		def 		A__					(				self):
			return self._lock_file


		@property
		def 		A__					(				self):
			return self._timeout


		@timeout.setter
		def 		A__					(				self	,       __snake_case):
			_UpperCamelCase   :		Dict			=						float(__snake_case)
			return None


		def 		A__					(				self):
			raise NotImplementedError()


		def 		A__					(				self):
			raise NotImplementedError()


		@property
		def 		A__					(				self):
			return self._lock_file_fd is not None


		def 		A__					(				self	,       __snake_case=None	,       __snake_case=0.0_5):
			# Use the default timeout, if no timeout is provided.
			if timeout is None:
				_UpperCamelCase   :		Tuple			=						self.timeout

			# Increment the number right at the beginning.
			# We can still undo it, if something fails.
			with self._thread_lock:
				self._lock_counter += 1

			_UpperCamelCase   :		Optional[int]			=						id(self)
			_UpperCamelCase   :		Optional[int]			=						self._lock_file
			_UpperCamelCase   :		Optional[int]			=						time.time()
			try:
				while True:
					with self._thread_lock:
						if not self.is_locked:
							logger().debug(f'''Attempting to acquire lock {lock_id} on {lock_filename}''')
							self._acquire()

					if self.is_locked:
						logger().debug(f'''Lock {lock_id} acquired on {lock_filename}''')
						break
					elif timeout >= 0 and time.time() - start_time > timeout:
						logger().debug(f'''Timeout on acquiring lock {lock_id} on {lock_filename}''')
						raise Timeout(self._lock_file)
					else:
						logger().debug(
						    f'''Lock {lock_id} not acquired on {lock_filename}, waiting {poll_intervall} seconds ...''')
						time.sleep(__snake_case)
			except:  # noqa
				# Something did go wrong, so decrement the counter.
				with self._thread_lock:
					_UpperCamelCase   :		Dict			=						max(0	,       self._lock_counter - 1)

				raise
			return _Acquire_ReturnProxy(lock=self)


		def 		A__					(				self	,       __snake_case=False):
			with self._thread_lock:
				if self.is_locked:
					self._lock_counter -= 1

					if self._lock_counter == 0 or force:
						_UpperCamelCase   :		Optional[Any]			=						id(self)
						_UpperCamelCase   :		List[str]			=						self._lock_file

						logger().debug(f'''Attempting to release lock {lock_id} on {lock_filename}''')
						self._release()
						_UpperCamelCase   :		List[Any]			=						0
						logger().debug(f'''Lock {lock_id} released on {lock_filename}''')

			return None


		def __enter__(				self):
			self.acquire()
			return self


		def __exit__(				self	,       __snake_case	,       __snake_case	,       __snake_case):
			self.release()
			return None


		def __del__(				self):
			self.release(force=__snake_case)
			return None






		def 		A__					(				self	,       __snake_case	,       __snake_case):
			_UpperCamelCase   :		Tuple			=						os.path.basename(__snake_case)
			if len(__snake_case) > max_length and max_length > 0:
				_UpperCamelCase   :		Optional[Any]			=						os.path.dirname(__snake_case)
				_UpperCamelCase   :		Union[str, Any]			=						str(hash(__snake_case))
				_UpperCamelCase   :		List[str]			=						filename[: max_length - len(__snake_case) - 8] + '...' + hashed_filename + '.lock'
				return os.path.join(__snake_case	,       __snake_case)
			else:
				return path







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		def __init__(				self	,       __snake_case	,       __snake_case=-1	,       __snake_case=None):
			from .file_utils import relative_to_absolute_path

			super().__init__(__snake_case	,       timeout=__snake_case	,       max_filename_length=__snake_case)
			_UpperCamelCase   :		Optional[int]			=						'\\\\?\\' + relative_to_absolute_path(self.lock_file)


		def 		A__					(				self):
			_UpperCamelCase   :		List[str]			=						os.O_RDWR | os.O_CREAT | os.O_TRUNC

			try:
				_UpperCamelCase   :		Dict			=						os.open(self._lock_file	,       __snake_case)
			except OSError:
				pass
			else:
				try:
					msvcrt.locking(__snake_case	,       msvcrt.LK_NBLCK	,       1)
				except OSError:
					os.close(__snake_case)
				else:
					_UpperCamelCase   :		List[Any]			=						fd
			return None






		def 		A__					(				self):
			_UpperCamelCase   :		str			=						self._lock_file_fd
			_UpperCamelCase   :		int			=						None
			msvcrt.locking(__snake_case	,       msvcrt.LK_UNLCK	,       1)
			os.close(__snake_case)

			try:
				os.remove(self._lock_file)
			# Probably another instance of the application
			# that acquired the file lock.
			except OSError:
				pass
			return None







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		def __init__(				self	,       __snake_case	,       __snake_case=-1	,       __snake_case=None):
			_UpperCamelCase   :		Union[str, Any]			=						os.statvfs(os.path.dirname(__snake_case)).f_namemax
			super().__init__(__snake_case	,       timeout=__snake_case	,       max_filename_length=__snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		List[str]			=						os.O_RDWR | os.O_CREAT | os.O_TRUNC
			_UpperCamelCase   :		Dict			=						os.open(self._lock_file	,       __snake_case)

			try:
				fcntl.flock(__snake_case	,       fcntl.LOCK_EX | fcntl.LOCK_NB)
			except OSError:
				os.close(__snake_case)
			else:
				_UpperCamelCase   :		Union[str, Any]			=						fd
			return None






		def 		A__					(				self):
			# Do not remove the lockfile:
			#
			#   https://github.com/benediktschmitt/py-filelock/issues/31
			#   https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
			_UpperCamelCase   :		Dict			=						self._lock_file_fd
			_UpperCamelCase   :		Union[str, Any]			=						None
			fcntl.flock(__snake_case	,       fcntl.LOCK_UN)
			os.close(__snake_case)
			return None







class 	lowercase    (				_lowercase							):


		"""simple docstring"""




		def 		A__					(				self):
			_UpperCamelCase   :		Tuple			=						os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
			try:
				_UpperCamelCase   :		Optional[int]			=						os.open(self._lock_file	,       __snake_case)
			except OSError:
				pass
			else:
				_UpperCamelCase   :		Dict			=						fd
			return None






		def 		A__					(				self):
			os.close(self._lock_file_fd)
			_UpperCamelCase   :		Optional[Any]			=						None

			try:
				os.remove(self._lock_file)
			# The file is already deleted and that's what we want.
			except OSError:
				pass
			return None

lowerCAmelCase__								=		None

if msvcrt:
							lowerCAmelCase__								=		WindowsFileLock
elif fcntl:
							lowerCAmelCase__								=		UnixFileLock
else:
							lowerCAmelCase__								=		SoftFileLock

							if warnings is not None:
														warnings.warn("""only soft file lock is available""")







import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
							import tensorflow as tf

if is_tensorflow_text_available():
							from transformers.models.bert import TFBertTokenizer


lowerCAmelCase__								=		["""bert-base-uncased""", """bert-base-cased"""]
lowerCAmelCase__								=		"""hf-internal-testing/tiny-bert-tf-only"""

if is_tf_available():







							class 	lowercase    (				tf.keras.Model							):


									"""simple docstring"""




									def __init__(				self	,       __snake_case):
										super().__init__()
										_UpperCamelCase   :		List[Any]			=						tokenizer
										_UpperCamelCase   :		List[Any]			=						AutoConfig.from_pretrained(__snake_case)
										_UpperCamelCase   :		Dict			=						TFAutoModel.from_config(__snake_case)






									def 		A__					(				self	,       __snake_case):
										_UpperCamelCase   :		Any			=						self.tokenizer(__snake_case)
										_UpperCamelCase   :		Dict			=						self.bert(**__snake_case)
										return out["pooler_output"]







@require_tf
@require_tensorflow_text
class 	lowercase    (				unittest.TestCase							):


		"""simple docstring"""




		def 		A__					(				self):
			super().setUp()

			_UpperCamelCase   :		Optional[Any]			=						[
			    BertTokenizer.from_pretrained(__snake_case) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
			]  # repeat for when fast_bert_tokenizer=false
			_UpperCamelCase   :		Optional[Any]			=						[TFBertTokenizer.from_pretrained(__snake_case) for checkpoint in TOKENIZER_CHECKPOINTS] + [
			    TFBertTokenizer.from_pretrained(__snake_case	,       use_fast_bert_tokenizer=__snake_case)
			    for checkpoint in TOKENIZER_CHECKPOINTS
			]
			assert len(self.tokenizers) == len(self.tf_tokenizers)

			_UpperCamelCase   :		Optional[Any]			=						[
			    'This is a straightforward English test sentence.',
			    'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
			    'Now we\'re going to add some Chinese: 一 二 三 一二三',
			    'And some much more rare Chinese: 齉 堃 齉堃',
			    'Je vais aussi écrire en français pour tester les accents',
			    'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
			]
			_UpperCamelCase   :		Dict			=						list(zip(self.test_sentences	,       self.test_sentences[::-1]))


		def 		A__					(				self):
			for tokenizer, tf_tokenizer in zip(self.tokenizers	,       self.tf_tokenizers):
				for test_inputs in (self.test_sentences, self.paired_sentences):
					_UpperCamelCase   :		List[str]			=						tokenizer(__snake_case	,       return_tensors='tf'	,       padding='longest')
					_UpperCamelCase   :		Tuple			=						tf_tokenizer(__snake_case)

					for key in python_outputs.keys():
						self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape))
						self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]	,       tf.intaa) == tf_outputs[key]))


		@slow
		def 		A__					(				self):
			for tf_tokenizer in self.tf_tokenizers:
				_UpperCamelCase   :		Tuple			=						tf_tokenizer(self.paired_sentences)
				_UpperCamelCase   :		Optional[Any]			=						tf_tokenizer(
				    text=[sentence[0] for sentence in self.paired_sentences]	,       text_pair=[sentence[1] for sentence in self.paired_sentences]	,       )
				for key in merged_outputs.keys():
					self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]	,       tf.intaa) == separated_outputs[key]))


		@slow
		def 		A__					(				self):
			for tf_tokenizer in self.tf_tokenizers:
				_UpperCamelCase   :		Tuple			=						tf.function(__snake_case)
				for test_inputs in (self.test_sentences, self.paired_sentences):
					_UpperCamelCase   :		Optional[int]			=						tf.constant(__snake_case)
					_UpperCamelCase   :		Union[str, Any]			=						compiled_tokenizer(__snake_case)
					_UpperCamelCase   :		Tuple			=						tf_tokenizer(__snake_case)

					for key in eager_outputs.keys():
						self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]))






		@slow
		def 		A__					(				self):
			for tf_tokenizer in self.tf_tokenizers:
				_UpperCamelCase   :		Any			=						ModelToSave(tokenizer=__snake_case)
				_UpperCamelCase   :		Any			=						tf.convert_to_tensor(self.test_sentences)
				_UpperCamelCase   :		Union[str, Any]			=						model(__snake_case)  # Build model with some sample inputs
				with TemporaryDirectory() as tempdir:
					_UpperCamelCase   :		int			=						Path(__snake_case) / 'saved.model'
					model.save(__snake_case)
					_UpperCamelCase   :		Optional[int]			=						tf.keras.models.load_model(__snake_case)
				_UpperCamelCase   :		int			=						loaded_model(__snake_case)
				# We may see small differences because the loaded model is compiled, so we need an epsilon for the test
				self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output))	,       1e-5)




