From efe9aef96e898670753621825c728375e987debd Mon Sep 17 00:00:00 2001 From: JackyYangPassion Date: Mon, 11 Mar 2024 16:56:01 +0800 Subject: [PATCH 1/2] init write HStore by pass server --- .../serializer/direct/HStoreSerializer.java | 128 ++++++++++++++++++ .../serializer/direct/reuse/BytesDemo.java | 28 ++-- 2 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java new file mode 100644 index 000000000..c25d4c4af --- /dev/null +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hugegraph.serializer.direct; + + +import org.apache.hugegraph.driver.HugeClient; +import org.apache.hugegraph.serializer.direct.struct.HugeType; +import org.apache.hugegraph.serializer.direct.util.BytesBuffer; +import org.apache.hugegraph.serializer.direct.util.GraphSchema; +import org.apache.hugegraph.serializer.direct.util.IdGenerator; +import org.apache.hugegraph.structure.GraphElement; +import org.apache.hugegraph.structure.graph.Edge; +import org.apache.hugegraph.structure.schema.PropertyKey; + +import java.util.Map; + +/** + * 直接实现BinarySerializer 生成点边Key-Value + */ +public class HStoreSerializer { + + private HugeClient client; + private GraphSchema graphSchema; + + public HStoreSerializer(HugeClient client) { + this.client = client; + this.graphSchema = new GraphSchema(client); + } + + public byte[] getKeyBytes(GraphElement e) { + byte[] array = null; + if (e.type() == "vertex" && e.id() != null) { + BytesBuffer buffer = BytesBuffer.allocate(2 + 1 + e.id().toString().length()); + //buffer.writeShort(getPartition(HugeType.VERTEX, IdGenerator.of(e.id()))); + buffer.writeId(IdGenerator.of(e.id())); + array = buffer.bytes(); + } else if (e.type() == "edge") { + BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); + Edge edge = (Edge)e; + //buffer.writeShort(getPartition(HugeType.EDGE, IdGenerator.of(edge.sourceId()))); + buffer.writeId(IdGenerator.of(edge.sourceId())); + buffer.write(HugeType.EDGE_OUT.code()); + buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); //出现错误 + buffer.writeStringWithEnding(""); + buffer.writeId(IdGenerator.of(edge.targetId())); + array = buffer.bytes(); + } + return array; + } + + public byte[] getKeyBytesSwitchDirection(GraphElement e) { + byte[] array = null; + BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); + Edge edge = (Edge)e; + //buffer.writeShort(getPartition(HugeType.EDGE, IdGenerator.of(edge.sourceId()))); + buffer.writeId(IdGenerator.of(edge.targetId())); + buffer.write(HugeType.EDGE_IN.code()); + buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); //出现错误 + buffer.writeStringWithEnding(""); + buffer.writeId(IdGenerator.of(edge.sourceId())); + array = buffer.bytes(); + return array; + } + + public byte[] getValueBytes(GraphElement e) { + byte[] array = null; + if (e.type() == "vertex") { + int propsCount = e.properties().size(); //vertex.sizeOfProperties(); + BytesBuffer buffer = BytesBuffer.allocate(8 + 16 * propsCount); + buffer.writeId(IdGenerator.of(graphSchema.getVertexLabel(e.label()).id())); + buffer.writeVInt(propsCount); + for (Map.Entry entry : e.properties().entrySet()) { + PropertyKey propertyKey = graphSchema.getPropertyKey(entry.getKey()); + buffer.writeVInt(propertyKey.id().intValue()); + buffer.writeProperty(propertyKey.dataType(),entry.getValue()); + } + array = buffer.bytes(); + } else if (e.type() == "edge") { + int propsCount = e.properties().size(); + BytesBuffer buffer = BytesBuffer.allocate(4 + 16 * propsCount); + buffer.writeVInt(propsCount); + for (Map.Entry entry : e.properties().entrySet()) { + PropertyKey propertyKey = graphSchema.getPropertyKey(entry.getKey()); + buffer.writeVInt(propertyKey.id().intValue()); + buffer.writeProperty(propertyKey.dataType(),entry.getValue()); + } + array = buffer.bytes(); + } + + return array; + } + + + + public byte[] getOwnerKeyBytes(GraphElement e) { + byte[] array = null; + if (e.type() == "vertex" && e.id() != null) { + array = IdGenerator.of(e.id()).asBytes(); + } else if (e.type() == "edge") { + Edge edge = (Edge)e; + array = IdGenerator.of(edge.sourceId()).asBytes(); + } + return array; + } + + + + + public void close() { + this.client.close(); + } + +} diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/reuse/BytesDemo.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/reuse/BytesDemo.java index ea7bbbd9a..36a3ea985 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/reuse/BytesDemo.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/reuse/BytesDemo.java @@ -25,12 +25,13 @@ import org.apache.hugegraph.driver.HugeClient; import org.apache.hugegraph.driver.SchemaManager; import org.apache.hugegraph.serializer.direct.HBaseSerializer; +import org.apache.hugegraph.serializer.direct.HStoreSerializer; import org.apache.hugegraph.serializer.direct.RocksDBSerializer; import org.apache.hugegraph.structure.graph.Edge; import org.apache.hugegraph.structure.graph.Vertex; /** - * This class is a demo for rocksdb/HBase put(rowkey, values) which use Client-Side's graph struct + * This class is a demo for Rocksdb/HBase/HStore put(rowkey, values) which use Client-Side's graph struct * And we don't need to construct the graph element, just use it and transfer them to bytes array * instead of json format */ @@ -39,7 +40,8 @@ public class BytesDemo { static HugeClient client; boolean bypassServer = true; RocksDBSerializer ser; - HBaseSerializer hBaseSer; + HStoreSerializer hStoreSerializer; + HBaseSerializer HBaseSer; public static void main(String[] args) { BytesDemo ins = new BytesDemo(); @@ -96,7 +98,7 @@ void initGraph() { .ifNotExist() .create(); - hBaseSer = new HBaseSerializer(client, vertexLogicPartitions, edgeLogicPartitions); + HBaseSer = new HBaseSerializer(client, vertexLogicPartitions, edgeLogicPartitions); writeGraphElements(); client.close(); @@ -146,14 +148,14 @@ private void writeGraphElements() { * */ void writeDirectly(List vertices, List edges) { for (Vertex vertex : vertices) { - byte[] rowkey = hBaseSer.getKeyBytes(vertex); - byte[] values = hBaseSer.getValueBytes(vertex); + byte[] rowkey = HBaseSer.getKeyBytes(vertex); + byte[] values = HBaseSer.getValueBytes(vertex); sendRpcToHBase("vertex", rowkey, values); } for (Edge edge : edges) { - byte[] rowkey = hBaseSer.getKeyBytes(edge); - byte[] values = hBaseSer.getValueBytes(edge); + byte[] rowkey = HBaseSer.getKeyBytes(edge); + byte[] values = HBaseSer.getValueBytes(edge); sendRpcToHBase("edge", rowkey, values); } } @@ -183,8 +185,18 @@ boolean sendRpcToHBase(String type, byte[] rowkey, byte[] values) { return flag; } + boolean sendRpcToHStore(String type, byte[] rowkey, byte[] values) { + boolean flag = false; + try { + flag = put(type, rowkey, values); + } catch (IOException e) { + e.printStackTrace(); + } + return flag; + } + boolean put(String type, byte[] rowkey, byte[] values) throws IOException { - // TODO: put to HBase + // TODO: put to HBase/HStore return true; } } From c1ec5e42455d45b19e2a93304d54c872d43feb5e Mon Sep 17 00:00:00 2001 From: JackyYangPassion Date: Mon, 11 Mar 2024 17:26:22 +0800 Subject: [PATCH 2/2] format code style --- .../serializer/direct/HStoreSerializer.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java index c25d4c4af..53f463fa1 100644 --- a/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java +++ b/hugegraph-client/src/main/java/org/apache/hugegraph/serializer/direct/HStoreSerializer.java @@ -17,7 +17,6 @@ package org.apache.hugegraph.serializer.direct; - import org.apache.hugegraph.driver.HugeClient; import org.apache.hugegraph.serializer.direct.struct.HugeType; import org.apache.hugegraph.serializer.direct.util.BytesBuffer; @@ -30,7 +29,7 @@ import java.util.Map; /** - * 直接实现BinarySerializer 生成点边Key-Value + * Directly implement BinarySerializer to generate point-edge Key-Value */ public class HStoreSerializer { @@ -46,16 +45,14 @@ public byte[] getKeyBytes(GraphElement e) { byte[] array = null; if (e.type() == "vertex" && e.id() != null) { BytesBuffer buffer = BytesBuffer.allocate(2 + 1 + e.id().toString().length()); - //buffer.writeShort(getPartition(HugeType.VERTEX, IdGenerator.of(e.id()))); buffer.writeId(IdGenerator.of(e.id())); array = buffer.bytes(); } else if (e.type() == "edge") { BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); Edge edge = (Edge)e; - //buffer.writeShort(getPartition(HugeType.EDGE, IdGenerator.of(edge.sourceId()))); buffer.writeId(IdGenerator.of(edge.sourceId())); buffer.write(HugeType.EDGE_OUT.code()); - buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); //出现错误 + buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); buffer.writeStringWithEnding(""); buffer.writeId(IdGenerator.of(edge.targetId())); array = buffer.bytes(); @@ -67,10 +64,9 @@ public byte[] getKeyBytesSwitchDirection(GraphElement e) { byte[] array = null; BytesBuffer buffer = BytesBuffer.allocate(BytesBuffer.BUF_EDGE_ID); Edge edge = (Edge)e; - //buffer.writeShort(getPartition(HugeType.EDGE, IdGenerator.of(edge.sourceId()))); buffer.writeId(IdGenerator.of(edge.targetId())); buffer.write(HugeType.EDGE_IN.code()); - buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); //出现错误 + buffer.writeId(IdGenerator.of(graphSchema.getEdgeLabel(e.label()).id())); buffer.writeStringWithEnding(""); buffer.writeId(IdGenerator.of(edge.sourceId())); array = buffer.bytes(); @@ -80,7 +76,7 @@ public byte[] getKeyBytesSwitchDirection(GraphElement e) { public byte[] getValueBytes(GraphElement e) { byte[] array = null; if (e.type() == "vertex") { - int propsCount = e.properties().size(); //vertex.sizeOfProperties(); + int propsCount = e.properties().size(); BytesBuffer buffer = BytesBuffer.allocate(8 + 16 * propsCount); buffer.writeId(IdGenerator.of(graphSchema.getVertexLabel(e.label()).id())); buffer.writeVInt(propsCount); @@ -101,12 +97,9 @@ public byte[] getValueBytes(GraphElement e) { } array = buffer.bytes(); } - return array; } - - public byte[] getOwnerKeyBytes(GraphElement e) { byte[] array = null; if (e.type() == "vertex" && e.id() != null) { @@ -118,9 +111,6 @@ public byte[] getOwnerKeyBytes(GraphElement e) { return array; } - - - public void close() { this.client.close(); }