Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Port token_model to TF2 #282

Open
wants to merge 2 commits into
base: users/boomanaiden154/main.port-token_model-to-tf2
Choose a base branch
from
Open

Port token_model to TF2 #282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
718cc4e
[𝘀𝗽𝗿] initial version
boomanaiden154 Jan 4, 2025
780467d
rebase
boomanaiden154 Jan 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions gematria/model/python/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,9 +185,6 @@ gematria_py_test(
size = "small",
timeout = "moderate",
srcs = ["token_model_test.py"],
tags = [
"manual",
],
deps = [
":oov_token_behavior",
":token_model",
Expand Down
23 changes: 11 additions & 12 deletions gematria/model/python/token_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,14 @@ def __init__(
if self._oov_token is None:
raise ValueError(f'Token {replacement_token} was not found in tokens.')

token_list_array = np.frombuffer(
b'\0'.join(token.encode('utf-8') for token in self._token_list),
dtype=np.uint8,
)
self._token_list_tensor = tf.constant(
token_list_array, name=TokenModel.TOKENS_TENSOR_NAME
)

super().__init__(**kwargs)

@property
Expand Down Expand Up @@ -195,16 +203,7 @@ def validate_basic_blockTokens(self, block: basic_block.BasicBlock) -> bool:
return False
return True

def _create_tf_graph(self):
def _forward(self, feed_dict: model_base.FeedDict):
"""See base class."""
super()._create_tf_graph()
# Convert the token list into an array of bytes. We need to go through NumPy
# because tf.constant() always treats a bytes() object as a string and can't
# use it with any other dtype.
token_list_array = np.frombuffer(
b'\0'.join(token.encode('utf-8') for token in self._token_list),
dtype=np.uint8,
)
self._token_list_tensor = tf.constant(
token_list_array, name=TokenModel.TOKENS_TENSOR_NAME
)
del feed_dict # Unused.
raise NotImplementedError()
7 changes: 2 additions & 5 deletions gematria/model/python/token_model_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from gematria.model.python import oov_token_behavior
from gematria.model.python import token_model
from gematria.testing.python import model_test
import tensorflow.compat.v1 as tf
import tensorflow as tf

_OutOfVocabularyTokenBehavior = oov_token_behavior.OutOfVocabularyTokenBehavior

Expand Down Expand Up @@ -66,9 +66,7 @@ def test_token_list_tensor(self):
model.initialize()

self.assertSequenceEqual(model._token_list, self.tokens)
with self.session() as sess:
raw_token_list = sess.run(model.token_list_tensor)
token_list = raw_token_list.tobytes().split(b'\0')
token_list = bytes(model.token_list_tensor).split(b'\0')
self.assertLen(token_list, len(set(token_list)))
for token in self.tokens:
self.assertIn(token.encode(), token_list)
Expand Down Expand Up @@ -167,5 +165,4 @@ def test_validate_basic_block_with_replacement_token(self):


if __name__ == '__main__':
tf.disable_v2_behavior()
tf.test.main()
Loading