-
Notifications
You must be signed in to change notification settings - Fork 599
HDDS-14561. SCMRatisRequest/ResponseProto should use the shaded protobuf from Ratis #9733
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
e87984a
c0b3145
4aa8930
eaa58a0
8eeaa6e
298f5de
ac43a62
2351cd1
df9720b
b1955b1
d2702dd
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,8 +17,8 @@ | |
|
|
||
| package org.apache.hadoop.hdds.scm.ha.io; | ||
|
|
||
| import com.google.protobuf.ByteString; | ||
| import com.google.protobuf.InvalidProtocolBufferException; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; | ||
|
|
||
| /** | ||
| * A dummy codec that serializes a ByteString object to ByteString. | ||
|
|
@@ -28,12 +28,28 @@ public class ByteStringCodec implements Codec { | |
| @Override | ||
| public ByteString serialize(Object object) | ||
| throws InvalidProtocolBufferException { | ||
| if (object instanceof ByteString) { | ||
| return (ByteString) object; | ||
| } | ||
| if (object instanceof com.google.protobuf.ByteString) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we still need to keep supporting com.google.protobuf.ByteString? Should everything used in SCM Codec have changed to org.apache.ratis.thirdparty.com.google.protobuf.ByteString after this PR? |
||
| com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) object; | ||
| return org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations | ||
| .unsafeWrap(bs.asReadOnlyByteBuffer()); | ||
| } | ||
| return (ByteString) object; | ||
| } | ||
|
|
||
| @Override | ||
| public Object deserialize(Class<?> type, ByteString value) | ||
| throws InvalidProtocolBufferException { | ||
| if (type == ByteString.class || ByteString.class.isAssignableFrom(type)) { | ||
| return value; | ||
| } | ||
| if (type == com.google.protobuf.ByteString.class | ||
| || com.google.protobuf.ByteString.class.isAssignableFrom(type)) { | ||
| return com.google.protobuf.UnsafeByteOperations | ||
| .unsafeWrap(value.asReadOnlyByteBuffer()); | ||
| } | ||
| return value; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,8 +17,6 @@ | |
|
|
||
| package org.apache.hadoop.hdds.scm.ha.io; | ||
|
|
||
| import com.google.protobuf.ByteString; | ||
| import com.google.protobuf.InvalidProtocolBufferException; | ||
| import com.google.protobuf.Message; | ||
| import com.google.protobuf.ProtocolMessageEnum; | ||
| import java.math.BigInteger; | ||
|
|
@@ -29,6 +27,7 @@ | |
| import java.util.Map; | ||
| import org.apache.commons.lang3.ClassUtils; | ||
| import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; | ||
|
|
||
| /** | ||
| * Maps types to the corresponding {@link Codec} implementation. | ||
|
|
@@ -47,7 +46,9 @@ public final class CodecFactory { | |
| codecs.put(Boolean.class, new BooleanCodec()); | ||
| codecs.put(BigInteger.class, new BigIntegerCodec()); | ||
| codecs.put(X509Certificate.class, new X509CertificateCodec()); | ||
| codecs.put(ByteString.class, new ByteStringCodec()); | ||
| codecs.put(com.google.protobuf.ByteString.class, new ByteStringCodec()); | ||
| codecs.put(org.apache.ratis.thirdparty.com.google.protobuf.ByteString.class, | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If com.google.protobuf.ByteString is still needed,
codecs.put(com.google.protobuf.ByteString.class, new ByteStringCodec());
codecs.put(ByteString.class, new ScmByteStringCodec());If com.google.protobuf.ByteString is not needed, just remove it. |
||
| new ByteStringCodec()); | ||
| codecs.put(ManagedSecretKey.class, new ManagedSecretKeyCodec()); | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,12 +18,12 @@ | |
| package org.apache.hadoop.hdds.scm.ha.io; | ||
|
|
||
| import com.google.common.primitives.Ints; | ||
| import com.google.protobuf.ByteString; | ||
| import com.google.protobuf.InvalidProtocolBufferException; | ||
| import com.google.protobuf.ProtoUtils; | ||
| import com.google.protobuf.ProtocolMessageEnum; | ||
| import java.lang.reflect.InvocationTargetException; | ||
| import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; | ||
|
|
||
| /** | ||
| * {@link Codec} for {@link ProtocolMessageEnum} objects. | ||
|
|
@@ -34,8 +34,7 @@ public class EnumCodec implements Codec { | |
| public ByteString serialize(Object object) | ||
| throws InvalidProtocolBufferException { | ||
| // toByteArray returns a new array | ||
| return ProtoUtils.unsafeByteString(Ints.toByteArray( | ||
| ((ProtocolMessageEnum) object).getNumber())); | ||
| return UnsafeByteOperations.unsafeWrap(Ints.toByteArray(((ProtocolMessageEnum) object).getNumber())); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FYI, filed HDDS-14623 for removing ProtoUtils. |
||
| } | ||
|
|
||
| @Override | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,34 +17,52 @@ | |
|
|
||
| package org.apache.hadoop.hdds.scm.ha.io; | ||
|
|
||
| import com.google.protobuf.ByteString; | ||
| import com.google.protobuf.InvalidProtocolBufferException; | ||
| import com.google.protobuf.Message; | ||
| import java.lang.reflect.InvocationTargetException; | ||
| import java.nio.ByteBuffer; | ||
| import org.apache.hadoop.hdds.scm.ha.ReflectionUtil; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.Message; | ||
|
|
||
| /** | ||
| * {@link Codec} for {@link Message} objects. | ||
| */ | ||
| public class GeneratedMessageCodec implements Codec { | ||
|
|
||
| @Override | ||
| public ByteString serialize(Object object) { | ||
| return ((Message)object).toByteString(); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we need to support both com.google.protobuf.Message and org.apache.ratis.thirdparty.com.google.protobuf.Message? If yes, use two GeneratedMessageCodec classes instead of using one class. Also, we need to put them to CodecFactory. //CodecFactory
codecs.put(com.google.protobuf.Message.class, new GeneratedMessageCodec());
codecs.put(Message.class, new ScmGeneratedMessageCodec()); |
||
| public ByteString serialize(Object object) throws InvalidProtocolBufferException { | ||
| try { | ||
| Object bs = ReflectionUtil | ||
| .getMethod(object.getClass(), "toByteString") | ||
| .invoke(object); | ||
| if (bs instanceof org.apache.ratis.thirdparty.com.google.protobuf.ByteString) { | ||
| return (org.apache.ratis.thirdparty.com.google.protobuf.ByteString) bs; | ||
| } | ||
| return org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations | ||
| .unsafeWrap(((com.google.protobuf.ByteString) bs).asReadOnlyByteBuffer()); | ||
|
|
||
| } catch (Exception e) { | ||
| throw new InvalidProtocolBufferException( | ||
| "Message cannot be encoded: " + e.getMessage()); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public Message deserialize(Class<?> type, ByteString value) | ||
| public Object deserialize(Class<?> type, ByteString value) | ||
| throws InvalidProtocolBufferException { | ||
| try { | ||
| return (Message) ReflectionUtil.getMethod(type, | ||
| "parseFrom", byte[].class) | ||
| .invoke(null, (Object) value.toByteArray()); | ||
| } catch (NoSuchMethodException | IllegalAccessException | ||
| | InvocationTargetException ex) { | ||
| ex.printStackTrace(); | ||
| try { | ||
| return ReflectionUtil | ||
| .getMethod(type, "parseFrom", ByteBuffer.class) | ||
| .invoke(null, value.asReadOnlyByteBuffer()); | ||
| } catch (NoSuchMethodException ignored) { | ||
| // fallback:parseFrom(byte[]) | ||
| return ReflectionUtil | ||
| .getMethod(type, "parseFrom", byte[].class) | ||
| .invoke(null, value.toByteArray()); | ||
| } | ||
| } catch (Exception e) { | ||
| throw new InvalidProtocolBufferException( | ||
| "Message cannot be decoded: " + ex.getMessage()); | ||
| "Message cannot be decoded: " + e.getMessage()); | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,10 +17,11 @@ | |
|
|
||
| package org.apache.hadoop.hdds.scm.ha.io; | ||
|
|
||
| import com.google.protobuf.ByteString; | ||
| import com.google.protobuf.InvalidProtocolBufferException; | ||
| import org.apache.hadoop.hdds.protocol.proto.SCMSecretKeyProtocolProtos; | ||
| import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException; | ||
| import org.apache.ratis.thirdparty.com.google.protobuf.UnsafeByteOperations; | ||
|
|
||
| /** | ||
| * A codec for {@link ManagedSecretKey} objects. | ||
|
|
@@ -30,14 +31,21 @@ public class ManagedSecretKeyCodec implements Codec { | |
| public ByteString serialize(Object object) | ||
| throws InvalidProtocolBufferException { | ||
| ManagedSecretKey secretKey = (ManagedSecretKey) object; | ||
| return secretKey.toProtobuf().toByteString(); | ||
| return UnsafeByteOperations.unsafeWrap( | ||
| secretKey.toProtobuf().toByteString().asReadOnlyByteBuffer()); | ||
| } | ||
|
|
||
| @Override | ||
| public Object deserialize(Class<?> type, ByteString value) | ||
| throws InvalidProtocolBufferException { | ||
| SCMSecretKeyProtocolProtos.ManagedSecretKey message = | ||
| SCMSecretKeyProtocolProtos.ManagedSecretKey.parseFrom(value); | ||
| return ManagedSecretKey.fromProtobuf(message); | ||
| try { | ||
| SCMSecretKeyProtocolProtos.ManagedSecretKey message = | ||
| SCMSecretKeyProtocolProtos.ManagedSecretKey.parseFrom( | ||
| value.asReadOnlyByteBuffer()); | ||
| return ManagedSecretKey.fromProtobuf(message); | ||
| } catch (com.google.protobuf.InvalidProtocolBufferException e) { | ||
| throw new org.apache.ratis.thirdparty.com.google.protobuf.InvalidProtocolBufferException( | ||
| e.getMessage()); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Let's include the type in the message and use e as the cause (it will keep e's stack trace.) throw new InvalidProtocolBufferException("Failed to deserialize value for " + type, e); |
||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@Russole , I see how this change works now -- it does prevent copying to an array but still needs type conversion from com.google.protobuf.ByteString to org.apache.ratis.thirdparty.com.google.protobuf.ByteString.
Let's fix also the conversion in this JIRA; see ContainerCommandRequestProto as an example for how to do it.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
unsafeWrap is not needed anymore.