-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Expand file tree
/
Copy pathtftext.patch
More file actions
89 lines (88 loc) · 4.39 KB
/
tftext.patch
File metadata and controls
89 lines (88 loc) · 4.39 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
diff --git a/tensorflow_text/core/kernels/fast_wordpiece_tokenizer.cc b/tensorflow_text/core/kernels/fast_wordpiece_tokenizer.cc
index 7f2c7c3..a9a6e06 100644
--- a/tensorflow_text/core/kernels/fast_wordpiece_tokenizer.cc
+++ b/tensorflow_text/core/kernels/fast_wordpiece_tokenizer.cc
@@ -34,7 +34,7 @@ namespace {
template <bool kGetPieces>
int GetCurrentOutputSize(std::vector<std::string>* output_pieces,
std::vector<int>* output_ids) {
- if constexpr (kGetPieces) {
+ if (kGetPieces) {
return output_pieces->size();
} else {
return output_ids->size();
@@ -557,10 +557,10 @@ void FastWordpieceTokenizer::AppendTokenToOutput(
std::vector<int>* output_end_offsets) const {
auto token_id =
fast_wordpiece_tokenizer_utils::GetTokenId(encoded_token_value);
- if constexpr (kGetIds) {
+ if (kGetIds) {
output_ids->push_back(token_id);
}
- if constexpr (kGetPieces || kGetOffsets) {
+ if (kGetPieces || kGetOffsets) {
// For suffix tokens, the length below is without the suffix indicator.
int token_substr_length =
fast_wordpiece_tokenizer_utils::GetTokenLength(encoded_token_value);
@@ -572,7 +572,7 @@ void FastWordpieceTokenizer::AppendTokenToOutput(
// to adjust and add the length of the suffix indicator string.
token_substr_length += config_->suffix_indicator()->size();
}
- if constexpr (kGetPieces) {
+ if (kGetPieces) {
// If token id is unk_token_id, it means that it is a dummy node for
// punctuations that are not contained in the vocabulary, we append
// the unk_token in this case. Otherwise, we
@@ -588,7 +588,7 @@ void FastWordpieceTokenizer::AppendTokenToOutput(
? absl::StrCat(config_->suffix_indicator()->str(), subword_str)
: subword_str);
}
- if constexpr (kGetOffsets) {
+ if (kGetOffsets) {
// Record the offsets relative to the start of the whole text.
output_start_offsets->push_back(input_word_offset_in_text +
cur_offset_in_input_word);
@@ -665,15 +665,15 @@ void FastWordpieceTokenizer::ResetOutputAppendUnknownToken(
std::vector<std::string>* output_pieces, std::vector<int>* output_ids,
std::vector<int>* output_start_offsets,
std::vector<int>* output_end_offsets) const {
- if constexpr (kGetPieces) {
+ if (kGetPieces) {
output_pieces->resize(original_num_tokens + 1);
output_pieces->back() = config_->unk_token()->str();
}
- if constexpr (kGetIds) {
+ if (kGetIds) {
output_ids->resize(original_num_tokens + 1);
output_ids->back() = config_->unk_token_id();
}
- if constexpr (kGetOffsets) {
+ if (kGetOffsets) {
output_start_offsets->resize(original_num_tokens + 1);
output_start_offsets->back() = input_word_offset_in_text;
diff --git a/tensorflow_text/tftext.bzl b/tensorflow_text/tftext.bzl
index 65430ca..e8584fb 100644
--- a/tensorflow_text/tftext.bzl
+++ b/tensorflow_text/tftext.bzl
@@ -144,8 +144,8 @@ def tf_cc_library(
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
"//conditions:default": [
- "@release_or_nightly//:tensorflow_libtensorflow_framework",
- "@release_or_nightly//:tensorflow_tf_header_lib",
+ "@org_tensorflow//tensorflow/core:tensorflow_opensource",
+ "@org_tensorflow//tensorflow/lite/kernels/shim:status_macros", "@org_tensorflow//tensorflow/lite/kernels/shim:tf_op_shim", "@org_tensorflow//tensorflow/lite/kernels/shim:op_kernel", "@org_tensorflow//tensorflow/lite/kernels/shim:tensor_view", "@org_tensorflow//tensorflow/lite/kernels/shim:shape",
] + tf_deps + oss_deps,
})
native.cc_library(
@@ -205,8 +205,8 @@ def tflite_cc_library(
"@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
],
"//conditions:default": [
- "@release_or_nightly//:tensorflow_libtensorflow_framework",
- "@release_or_nightly//:tensorflow_tf_header_lib",
+ "@org_tensorflow//tensorflow/core:tensorflow_opensource",
+ "@org_tensorflow//tensorflow/lite/kernels/shim:status_macros", "@org_tensorflow//tensorflow/lite/kernels/shim:tf_op_shim", "@org_tensorflow//tensorflow/lite/kernels/shim:op_kernel", "@org_tensorflow//tensorflow/lite/kernels/shim:tensor_view", "@org_tensorflow//tensorflow/lite/kernels/shim:shape",
] + oss_deps,
})
native.cc_library(