



'''simple docstring'''


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_      :				Any   					=    get_tests_dir('''fixtures/spiece.model''')




@require_sentencepiece
@require_tokenizers
class      __UpperCamelCase       (_UpperCAmelCase			,   unittest.TestCase			):
 __A			  =					AlbertTokenizer
 __A			  =					AlbertTokenizerFast
 __A			  =					True
 __A			  =					True
 __A			  =					True







 def   _a   (    self						)    ->    int:
       '''simple docstring'''






       super().setUp()

       # We have a SentencePiece fixture for testing
       lowercase        =							AlbertTokenizer(_lowerCAmelCase						)
       tokenizer.save_pretrained(self.tmpdirname						)







 def   _a   (    self ,				_lowerCAmelCase						)    ->    Any:
       '''simple docstring'''






       lowercase        =							"""this is a test"""
       lowercase        =							"""this is a test"""
       return input_text, output_text







 def   _a   (    self						)    ->    Optional[int]:
       '''simple docstring'''






       lowercase        =							"""<pad>"""
       lowercase        =							0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase						) ,				_lowerCAmelCase						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase						) ,				_lowerCAmelCase						)







 def   _a   (    self						)    ->    Dict:
       '''simple docstring'''






       lowercase        =							list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0] ,				"""<pad>"""						)
       self.assertEqual(vocab_keys[1] ,				"""<unk>"""						)
       self.assertEqual(vocab_keys[-1] ,				"""▁eloquent"""						)
       self.assertEqual(len(_lowerCAmelCase						) ,				3_0000						)







 def   _a   (    self						)    ->    Optional[Any]:
       '''simple docstring'''






       self.assertEqual(self.get_tokenizer().vocab_size ,				3_0000						)







 def   _a   (    self						)    ->    List[str]:
       '''simple docstring'''






       if not self.test_rust_tokenizer:
             return

       lowercase        =							self.get_tokenizer()
       lowercase        =							self.get_rust_tokenizer()

       lowercase        =							"""I was born in 92000, and this is falsé."""

       lowercase        =							tokenizer.tokenize(_lowerCAmelCase						)
       lowercase        =							rust_tokenizer.tokenize(_lowerCAmelCase						)
       self.assertListEqual(_lowerCAmelCase ,				_lowerCAmelCase						)

       lowercase        =							tokenizer.encode(_lowerCAmelCase ,				add_special_tokens=_lowerCAmelCase						)
       lowercase        =							rust_tokenizer.encode(_lowerCAmelCase ,				add_special_tokens=_lowerCAmelCase						)
       self.assertListEqual(_lowerCAmelCase ,				_lowerCAmelCase						)

       lowercase        =							self.get_rust_tokenizer()
       lowercase        =							tokenizer.encode(_lowerCAmelCase						)
       lowercase        =							rust_tokenizer.encode(_lowerCAmelCase						)
       self.assertListEqual(_lowerCAmelCase ,				_lowerCAmelCase						)







 def   _a   (    self						)    ->    Optional[int]:
       '''simple docstring'''






       lowercase        =							AlbertTokenizer(_lowerCAmelCase ,				keep_accents=_lowerCAmelCase						)

       lowercase        =							tokenizer.tokenize("""This is a test"""						)
       self.assertListEqual(_lowerCAmelCase ,				["""▁this""", """▁is""", """▁a""", """▁test"""]						)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase						) ,				[48, 25, 21, 1289]						)

       lowercase        =							tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
       self.assertListEqual(
           _lowerCAmelCase ,				["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """é""", """."""]						)
       lowercase        =							tokenizer.convert_tokens_to_ids(_lowerCAmelCase						)
       self.assertListEqual(_lowerCAmelCase ,				[31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]						)

       lowercase        =							tokenizer.convert_ids_to_tokens(_lowerCAmelCase						)
       self.assertListEqual(
           _lowerCAmelCase ,				["""▁i""", """▁was""", """▁born""", """▁in""", """▁9""", """2000""", """,""", """▁and""", """▁this""", """▁is""", """▁fal""", """s""", """<unk>""", """."""] ,				)







 def   _a   (    self						)    ->    List[Any]:
       '''simple docstring'''






       lowercase        =							AlbertTokenizer(_lowerCAmelCase						)

       lowercase        =							tokenizer.encode("""sequence builders"""						)
       lowercase        =							tokenizer.encode("""multi-sequence build"""						)

       lowercase        =							tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase						)
       lowercase        =							tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase ,				_lowerCAmelCase						)

       assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
       assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
           tokenizer.sep_token_id
       ]







 @slow
 def   _a   (    self						)    ->    List[Any]:
       '''simple docstring'''






       lowercase        =							{"""attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """input_ids""": [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=_lowerCAmelCase ,				model_name="""albert-base-v2""" ,				revision="""6b6560eaf5ff2e250b00c50f380c5389a9c2d82e""" ,				)







'''simple docstring'''


from random import shuffle

import tensorflow as tf
from numpy import array







def      SCREAMING_SNAKE_CASE				(				lowercase_    :      List[str] ,  lowercase_    :      Optional[int]						):
						lowercase        =							int(lowercase_						)
						assert noofclusters < len(lowercase_						)

						# Find out the dimensionality
						lowercase        =							len(vectors[0]						)

						# Will help select random centroids from among the available vectors
						lowercase        =							list(range(len(lowercase_						)						)						)
						shuffle(lowercase_						)

						# GRAPH OF COMPUTATION
						# We initialize a new graph and set it as the default during each run
						# of this algorithm. This ensures that as this function is called
						# multiple times, the default graph doesn't keep getting crowded with
						# unused ops and Variables from previous function calls.

						lowercase        =							tf.Graph()

						with graph.as_default():
												# SESSION OF COMPUTATION

												lowercase        =							tf.Session()

												##CONSTRUCTING THE ELEMENTS OF COMPUTATION

												##First lets ensure we have a Variable vector for each centroid,
												##initialized to one of the vectors from the available data points
												lowercase        =							[
												    tf.Variable(vectors[vector_indices[i]]						) for i in range(lowercase_						)
												]
												##These nodes will assign the centroid Variables the appropriate
												##values
												lowercase        =							tf.placeholder("""float64""" ,  [dim]						)
												lowercase        =							[]
												for centroid in centroids:
																		cent_assigns.append(tf.assign(lowercase_ ,  lowercase_						)						)

												##Variables for cluster assignments of individual vectors(initialized
												##to 0 at first)
												lowercase        =							[tf.Variable(0						) for i in range(len(lowercase_						)						)]
												##These nodes will assign an assignment Variable the appropriate
												##value
												lowercase        =							tf.placeholder("""int32"""						)
												lowercase        =							[]
												for assignment in assignments:
																		cluster_assigns.append(tf.assign(lowercase_ ,  lowercase_						)						)

												##Now lets construct the node that will compute the mean
												# The placeholder for the input
												lowercase        =							tf.placeholder("""float""" ,  [None, dim]						)
												# The Node/op takes the input and computes a mean along the 0th
												# dimension, i.e. the list of input vectors
												lowercase        =							tf.reduce_mean(lowercase_ ,  0						)

												##Node for computing Euclidean distances
												# Placeholders for input
												lowercase        =							tf.placeholder("""float""" ,  [dim]						)
												lowercase        =							tf.placeholder("""float""" ,  [dim]						)
												lowercase        =							tf.sqrt(tf.reduce_sum(tf.pow(tf.sub(lowercase_ ,  lowercase_						) ,  2						)						)						)

												##This node will figure out which cluster to assign a vector to,
												##based on Euclidean distances of the vector from the centroids.
												# Placeholder for input
												lowercase        =							tf.placeholder("""float""" ,  [noofclusters]						)
												lowercase        =							tf.argmin(lowercase_ ,  0						)

												##INITIALIZING STATE VARIABLES

												##This will help initialization of all Variables defined with respect
												##to the graph. The Variable-initializer should be defined after
												##all the Variables have been constructed, so that each of them
												##will be included in the initialization.
												lowercase        =							tf.initialize_all_variables()

												# Initialize all variables
												sess.run(lowercase_						)

												##CLUSTERING ITERATIONS

												# Now perform the Expectation-Maximization steps of K-Means clustering
												# iterations. To keep things simple, we will only do a set number of
												# iterations, instead of using a Stopping Criterion.
												lowercase        =							100
												for _ in range(lowercase_						):
																		##EXPECTATION STEP
																		##Based on the centroid locations till last iteration, compute
																		##the _expected_ centroid assignments.
																		# Iterate over each vector
																		for vector_n in range(len(lowercase_						)						):
																								lowercase        =							vectors[vector_n]
																								# Compute Euclidean distance between this vector and each
																								# centroid. Remember that this list cannot be named
																								#'centroid_distances', since that is the input to the
																								# cluster assignment node.
																								lowercase        =							[
																								    sess.run(lowercase_ ,  feed_dict={va: vect, va: sess.run(lowercase_						)}						)
																								    for centroid in centroids
																								]
																								# Now use the cluster assignment node, with the distances
																								# as the input
																								lowercase        =							sess.run(
																								    lowercase_ ,  feed_dict={centroid_distances: distances}						)
																								# Now assign the value to the appropriate state variable
																								sess.run(
																								    cluster_assigns[vector_n] ,  feed_dict={assignment_value: assignment}						)

																		##MAXIMIZATION STEP
																		# Based on the expected state computed from the Expectation Step,
																		# compute the locations of the centroids so as to maximize the
																		# overall objective of minimizing within-cluster Sum-of-Squares
																		for cluster_n in range(lowercase_						):
																								# Collect all the vectors assigned to this cluster
																								lowercase        =							[
																								    vectors[i]
																								    for i in range(len(lowercase_						)						)
																								    if sess.run(assignments[i]						) == cluster_n
																								]
																								# Compute new centroid location
																								lowercase        =							sess.run(
																								    lowercase_ ,  feed_dict={mean_input: array(lowercase_						)}						)
																								# Assign value to appropriate variable
																								sess.run(
																								    cent_assigns[cluster_n] ,  feed_dict={centroid_value: new_location}						)

        # Return centroids and assignments
												lowercase        =							sess.run(lowercase_						)
												lowercase        =							sess.run(lowercase_						)
												return centroids, assignments


