diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1d25004..90b2353 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -13,6 +13,7 @@ on: branches: [ "master" ] pull_request: branches: [ "master" ] + workflow_dispatch: jobs: build: @@ -20,15 +21,47 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: maven - - name: Build with Maven - run: mvn -B package --file pom.xml + - name: Checkout project + uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build with Maven + run: mvn -B package -DskipTests -Prelease --file pom.xml + + - name: Set up Apache Maven Central + uses: actions/setup-java@v3 + with: # running setup-java again overwrites the settings.xml + java-version: '11' + distribution: 'temurin' + server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml + server-username: MAVEN_USERNAME # env variable for username in deploy + server-password: MAVEN_PASSWORD # env variable for token in deploy + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import + gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + + - name: Publish to the Maven Central Repository + run: + mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests \ + deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am + env: + MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive # - name: Update dependency graph diff --git a/highflip-build/pom.xml b/highflip-build/pom.xml index 6d5cf4e..5ae3101 100644 --- a/highflip-build/pom.xml +++ b/highflip-build/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-build jar 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-console/pom.xml b/highflip-clients/highflip-console/pom.xml index 3047fff..1815e6d 100644 --- a/highflip-clients/highflip-console/pom.xml +++ b/highflip-clients/highflip-console/pom.xml @@ -11,16 +11,16 @@ --> - com.baidu + com.baidu.highflip highflip-console 1.0.0-SNAPSHOT highflip-console - com.baidu - highflip + com.baidu.highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml @@ -44,7 +44,7 @@ spring-boot-starter-json - com.baidu + com.baidu.highflip highflip-sdk 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java index 724295b..5c969f0 100644 --- a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java +++ b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java @@ -17,12 +17,13 @@ public class TaskCommand { @Autowired HighFlipClient client; - @ShellMethod(key = "task list", value = "List all task ids") + @ShellMethod(key = "task list", value = "List job all task ids") public Iterable list( + @ShellOption String jobId, @ShellOption(defaultValue = "0") Integer offset, @ShellOption(defaultValue = "0") Integer limit) { - return client.listTasks(offset, limit); + return client.listTasks(jobId, offset, limit); } @ShellMethod(key = "task get", value = "Get a task information") diff --git a/highflip-clients/highflip-sdk/pom.xml b/highflip-clients/highflip-sdk/pom.xml index d982e95..49e9514 100644 --- a/highflip-clients/highflip-sdk/pom.xml +++ b/highflip-clients/highflip-sdk/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-sdk jar 1.0.0-SNAPSHOT @@ -10,17 +10,17 @@ https://maven.apache.org - com.baidu - highflip + com.baidu.highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml - com.baidu + com.baidu.highflip highflip-proto - 1.0.0-SNAPSHOT + ${project.version} org.projectlombok diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 9a57d8c..3cd80e0 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -42,6 +42,12 @@ public HighFlipClient() { this.blockingStub = null; } + public static HighFlipClient newHighFlipClient(String target) { + HighFlipClient highFlipClient = new HighFlipClient(); + highFlipClient.connect(target); + return highFlipClient; + } + public void connect(String target) { close(); @@ -283,13 +289,17 @@ public Iterable getJobLog(String jobId){ /** * + * @param jobId job id in highflip * @param offset * @param limit * @return */ - public Iterable listTasks(int offset, int limit){ + public Iterable listTasks(String jobId, int offset, int limit) { Highflip.TaskListRequest request = Highflip.TaskListRequest .newBuilder() + .setJobId(jobId) + .setOffset(offset) + .setLimit(limit) .build(); Iterator response = getBlockingStub() diff --git a/highflip-clients/pom.xml b/highflip-clients/pom.xml index 3de5034..1457d0f 100644 --- a/highflip-clients/pom.xml +++ b/highflip-clients/pom.xml @@ -4,13 +4,13 @@ highflip - com.baidu + com.baidu.highflip 1.0.0-SNAPSHOT ../pom.xml - com.baidu + com.baidu.highflip highflip-clients 1.0.0-SNAPSHOT pom diff --git a/highflip-core/pom.xml b/highflip-core/pom.xml index ff39144..22b019e 100644 --- a/highflip-core/pom.xml +++ b/highflip-core/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-core jar 1.0.0-SNAPSHOT @@ -11,7 +11,7 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml @@ -19,7 +19,7 @@ - com.baidu + com.baidu.highflip highflip-proto 1.0.0-SNAPSHOT diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java new file mode 100644 index 0000000..104cfc1 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java @@ -0,0 +1,10 @@ +package com.baidu.highflip.core.adaptor; + +public interface ServiceAdaptor { + + String getUrl(); + + String getPartyId(); + + String getRole(); +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java index 70522ff..f156486 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java @@ -25,4 +25,16 @@ public class AdaptorPropsList { public static final String PROPS_HIGHFLIP_ADAPTOR_PLATFORM_VERSION_DEFAULT = "0.0.0"; + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL = "highflip.adaptor.service.url"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT = "http://127.0.0.1:9380"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID = "highflip.adaptor.service.party.id"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT = "9999"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE = "highflip.adaptor.service.role"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT = "guest"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java index ef166ec..1a34c7f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java @@ -25,4 +25,7 @@ public class InstanceNameList { public static final String HIGHFLIP_ADAPTOR_PARTNER = "highflip.adaptor.partner"; public static final String HIGHFLIP_ADAPTOR_USER = "highflip.adaptor.user"; + + public static final String HIGHFLIP_ADAPTOR_SERVICE = "highflip.adaptor.service"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java index 26f069a..21a56a6 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java @@ -6,6 +6,7 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; public interface HighFlipRuntime { @@ -26,4 +27,10 @@ public interface HighFlipRuntime { User getUser(String userId); Operator getOperator(String operatorId); + + Data registerData(Data data); + + Iterable listTask(String jobId); + + void updateTask(Task task); } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java index 2224ee4..73eacfe 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java @@ -3,6 +3,8 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; import lombok.NoArgsConstructor; @@ -96,7 +98,7 @@ public Iterable listParties() { return getParties().keySet(); } - + @JsonIgnore protected void setNodeCategory() { calcMiddleNodes(); calcOutputNodes(); diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java index 2089128..78d73c0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java @@ -6,16 +6,19 @@ import com.baidu.highflip.core.entity.dag.common.NodeInputRef; import com.baidu.highflip.core.entity.dag.common.NodeOutputRef; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; -import javax.persistence.Transient; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.springframework.data.annotation.Transient; + @Data public class Node extends NamedAttributeObject implements Serializable { @@ -25,7 +28,7 @@ public class Node extends NamedAttributeObject implements Serializable { String description; - @Transient + @JsonIgnore Graph graph; Category category; @@ -108,6 +111,7 @@ public Node getInputNode(String name) { getInputs().get(name).getFromNode()); } + @JsonIgnore public List getInputNodes() { return getInputs() .values() diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java index c96e552..5d096f0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java @@ -3,6 +3,7 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; + import highflip.HighflipMeta; import lombok.Data; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java index b5227f4..1a43401 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java @@ -1,5 +1,12 @@ package com.baidu.highflip.core.entity.dag.codec; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.protobuf.ByteString; + import highflip.HighflipMeta; public class TypeValue { @@ -17,11 +24,76 @@ public static Object fromProto(HighflipMeta.TypedValueProto value) { return value.getValue().getFloat(); } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.STRING.getNumber()) { return value.getValue().getString(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.BYTES.getNumber()) { + return value.getValue().getBytes(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.LIST.getNumber()) { + return listProtoConvert2List(value.getValue().getList()); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.MAP.getNumber()) { + return mapProtoConvert2Map(value.getValue().getMap()); } else { throw new IllegalArgumentException(); } } + private static List listProtoConvert2List( + HighflipMeta.ListProto listProto) { + if (listProto == null) { + return new ArrayList<>(); + } + List result = new ArrayList<>(); + List listProtoList = listProto.getListList(); + for (HighflipMeta.ValueProto valueProto : listProtoList) { + result.add(convertValueProto(valueProto)); + } + return result; + } + + private static Map mapProtoConvert2Map(HighflipMeta.MapProto mapProto) { + if (mapProto == null) { + return new HashMap<>(); + } + Map map = mapProto.getMapMap(); + Map result = new HashMap<>(); + for (Map.Entry entry : + map.entrySet()) { + String key = entry.getKey(); + HighflipMeta.ValueProto value = entry.getValue(); + result.put(key, convertValueProto(value)); + } + return result; + } + + private static Object convertValueProto(HighflipMeta.ValueProto value) { + switch (value.getValueCase()) { + case BOOL: + return value.getBool(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + case STRING: + return value.getString(); + case BYTES: + return value.getBytes(); + case LIST: + List list = + value.getList().getListList(); + List resultList = new ArrayList<>(); + for(HighflipMeta.ValueProto listValue: list) { + resultList.add(convertValueProto(listValue)); + } + return resultList; + case MAP: + return mapProtoConvert2Map(value.getMap()); + default: + throw new RuntimeException("NOT_SUPPORTED_TYPE"); + } + } + public static HighflipMeta.TypedValueProto toProto(Object object) { HighflipMeta.TypedValueProto.Builder builder = HighflipMeta.TypedValueProto .newBuilder(); @@ -56,14 +128,31 @@ public static HighflipMeta.TypedValueProto toProto(Object object) { .newBuilder() .setDouble((Double) object) .build()); - } else { + } else if (object instanceof String) { builder.setType(HighflipMeta.TypedValueProto.TypeProto.STRING); builder.setValue(HighflipMeta.ValueProto .newBuilder() .setString(object.toString()) .build()); + } else if (object instanceof ByteString) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.BYTES); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setBytes((ByteString) object) + .build()); + } else if (object instanceof HighflipMeta.ListProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.LIST); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setList((HighflipMeta.ListProto) object) + .build()); + } else if (object instanceof HighflipMeta.MapProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.MAP); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setMap((HighflipMeta.MapProto) object) + .build()); } - return builder.build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java index e0f1377..0b9f14a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java @@ -2,15 +2,18 @@ import lombok.Data; -import javax.persistence.Transient; import java.util.List; import java.util.Map; import java.util.TreeMap; +import org.springframework.data.annotation.Transient; + +import com.fasterxml.jackson.annotation.JsonIgnore; + @Data public class NamedAttributeObject implements Comparable { - @Transient + @JsonIgnore List parents = List.of(); String name; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java index a6d9a63..d6b2eef 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java @@ -19,12 +19,15 @@ public class NodeInputRef { String fromOutput; + String value; + public static HighflipMeta.NodeInputProto toProto(com.baidu.highflip.core.entity.dag.common.NodeInputRef ref) { HighflipMeta.NodeInputProto.Builder builder = HighflipMeta.NodeInputProto .newBuilder() .setName(ref.getName()) .setFromNode(ref.getFromNode()) - .setFromOutput(ref.getFromOutput()); + .setFromOutput(ref.getFromOutput()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -37,6 +40,7 @@ public static com.baidu.highflip.core.entity.dag.common.NodeInputRef fromProto(H .setDescription(proto.getDescription()) .setFromNode(proto.getFromNode()) .setFromOutput(proto.getFromOutput()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java index e9fd012..270288a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java @@ -15,10 +15,13 @@ public class NodeOutputRef { String description; + String value; + public static HighflipMeta.NodeOutputProto toProto(NodeOutputRef ref) { HighflipMeta.NodeOutputProto.Builder builder = HighflipMeta.NodeOutputProto .newBuilder() - .setName(ref.getName()); + .setName(ref.getName()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -29,6 +32,7 @@ public static NodeOutputRef fromProto(HighflipMeta.NodeOutputProto proto) { .builder() .setName(proto.getName()) .setDescription(proto.getDescription()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java index 8086c0a..bc4239b 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java @@ -33,7 +33,7 @@ public class Data { @Id @Column(name = "data_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String dataId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java index 9d506fc..aaf9d3f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java @@ -37,7 +37,7 @@ public class Job { @Id @Column(name = "job_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String jobId; @@ -59,7 +59,7 @@ public class Job { DateTime finishTime; @Type(type = "json") - @Column(name = "graph") + @Column(name = "graph", length = 10240) Graph graph; @Column(name = "status") diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java index d8e9054..1af34ec 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java @@ -28,7 +28,7 @@ public class Operator { @Id @Column(name = "operator_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String operatorId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java index 41f25f2..6ab0f5a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java @@ -27,7 +27,7 @@ public class Partner { @Id @Column(name = "partner_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String partnerId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java index db5eaf6..8d7936a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java @@ -33,7 +33,7 @@ public class Platform { @Id @Column(name = "platform_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String platformId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java index 07c3e35..f4ed6ad 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java @@ -31,10 +31,14 @@ public class Task { @Id @Column(name = "task_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String taskid; + /** + * job id is not the job id in highflip but the job id in corresponding + * federated learning service + */ @Column(name = "job_id") String jobid; @@ -81,4 +85,9 @@ public class Task { @Type(type = "json") @Column(name = "binding") Map binding; + + @Type(type = "json") + @Column(name = "output_data") + List outputData; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java new file mode 100644 index 0000000..4e9ea66 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java @@ -0,0 +1,24 @@ +package com.baidu.highflip.core.utils; + +import java.io.Serializable; + +import org.hibernate.HibernateException; +import org.hibernate.engine.spi.SharedSessionContractImplementor; +import org.hibernate.id.UUIDGenerator; + +public class CustomUuidGenerator extends UUIDGenerator { + + @Override + public Serializable generate(SharedSessionContractImplementor s, Object obj) + throws HibernateException { + Serializable id = s.getEntityPersister(null, obj).getClassMetadata() + .getIdentifier(obj, s); + + if (id != null && String.valueOf(id).length() > 0) { + return id; + } else { + return super.generate(s, obj); + } + } + +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java new file mode 100644 index 0000000..d252f4a --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java @@ -0,0 +1,64 @@ +package com.baidu.highflip.core.utils; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.springframework.lang.Nullable; + +public class SerializerUtils { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private SerializerUtils() { + } + + public static Map mapObjectDeserialize(String object, + Class tClass) + throws JsonProcessingException { + if (object == null || object.isBlank()) { + return null; + } + Map ret = new HashMap<>(); + Map map = objectMapper.readValue(object, + new TypeReference>() { + }); + for (String tempKey : map.keySet()) { + Object tempValue = map.get(tempKey); + if (tempValue != null) { + String decodeValue = objectMapper.writeValueAsString(tempValue); + if (tClass.isAssignableFrom(String.class)) { + //noinspection unchecked + ret.put(tempKey, (T) decodeValue); + } else { + T result = objectMapper.readValue(decodeValue, tClass); + ret.put(tempKey, result); + } + } + } + return ret; + } + + public static String toJsonString(Object object) + throws JsonProcessingException { + return objectMapper.writeValueAsString(object); + } + + public static T deserialize(String json, Class tClass) + throws JsonProcessingException { + return objectMapper.readValue(json, tClass); + } + + @Nullable + public static T deserializeType(String json, + TypeReference typeReference) + throws JsonProcessingException { + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, typeReference); + } +} diff --git a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java index 7584d1a..b7f2135 100644 --- a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java +++ b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java @@ -6,12 +6,18 @@ import com.baidu.highflip.core.entity.dag.PartyNode; import com.baidu.highflip.core.entity.dag.codec.TypeValue; import com.baidu.highflip.core.entity.dag.common.MappedNode; +import com.baidu.highflip.core.utils.SerializerUtils; +import com.fasterxml.jackson.core.JsonProcessingException; + import highflip.HighflipMeta; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; @Slf4j @@ -19,10 +25,7 @@ public class TestDag { @Test public void testGraph() { - HighflipMeta.GraphProto define = HighflipMeta.GraphProto - .newBuilder() - .setName("test_graph") - .build(); + HighflipMeta.GraphProto define = constructHighflipGraph(); var graph = Graph.fromProto(define); log.info("graph = {}", graph); @@ -229,4 +232,353 @@ public void testDag() { var proto = Graph.toProto(graph); log.info("proto = {}", proto); } + + private HighflipMeta.GraphProto constructHighflipGraph() { + // reader + List readerNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto outputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + readerNodeOutputProtos.add(outputProto); + + HighflipMeta.NodeProto reader = + HighflipMeta.NodeProto.newBuilder().setName("reader_0") + .setType("Reader") + .addAllOutputs(readerNodeOutputProtos) + .build(); + + // DataTransform + List dataTransformNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto dataTransformOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + HighflipMeta.NodeOutputProto dataTransformOutputProto2 = + HighflipMeta.NodeOutputProto.newBuilder().setName("model").setValue(getJsonStringOfModelOutput()).build(); + dataTransformNodeOutputProtos.add(dataTransformOutputProto); + dataTransformNodeOutputProtos.add(dataTransformOutputProto2); + + + + List dataTransformNodeInputProtos = + new ArrayList<>(); + String json = getDataTransformInputJsonString(); + HighflipMeta.NodeInputProto dataTransformInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(json) + .build(); + + + dataTransformNodeInputProtos.add(dataTransformInputProto); + + HighflipMeta.NodeProto dataTransform = + HighflipMeta.NodeProto.newBuilder().setName("data_transform_0") + .setType("DataTransform") + .addAllInputs( + dataTransformNodeInputProtos) + .addAllOutputs( + dataTransformNodeOutputProtos) + .build(); + + // intersect + List intersectNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto intersectOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + intersectNodeOutputProtos.add(intersectOutputProto); + + List intersectNodeInputProtos = + new ArrayList<>(); + + String dataTransformInputJsonString = getIntersectionInputJsonString(); + HighflipMeta.NodeInputProto intersectInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(dataTransformInputJsonString) + .build(); + + intersectNodeInputProtos.add(intersectInputProto); + + HighflipMeta.NodeProto intersect = + HighflipMeta.NodeProto.newBuilder().setName("intersect_0") + .setType("Intersection") + .addAllInputs(intersectNodeInputProtos) + .addAllOutputs(intersectNodeOutputProtos) + .build(); + + + // Map tableMap = new HashMap<>(); + // tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + // tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto mapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + // HighflipMeta.ValueProto valueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + // highflip.HighflipMeta.TypedValueProto typedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getGuestPartyReaderNode()) + .build(); + // + HighflipMeta.TypedValueProto withLabel= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + + HighflipMeta.TypedValueProto outputFormat= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("dense") + .build()).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + // common conf + /* + + "intersect_0": { + "intersect_method": "rsa", + "sync_intersect_ids": false, + "only_output_key": true, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "split_calculation": false, + "key_length": 2048 + } + } + */ + HighflipMeta.TypedValueProto intersectMethod= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("rsa") + .build()).build(); + HighflipMeta.TypedValueProto syncIntersectIds= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto onlyOutputKey= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(true) + .build()).build(); + + // Map commonTableMap = + // new HashMap<>(); + + // tableMap.put("hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("split_calculation", HighflipMeta.ValueProto.newBuilder().setBool(false).build()); + // tableMap.put("key_length", HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + + + + + // Map commonRsaParamConfMap = + // new HashMap<>(); + // commonRsaParamConfMap.put("hash_method", + // HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("split_calculation", + // HighflipMeta.ValueProto.newBuilder().setBool( + // false).build()); + // commonRsaParamConfMap.put("key_length", + // HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + // HighflipMeta.MapProto commonValueMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + // HighflipMeta.ValueProto commonValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + // highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + + HighflipMeta.PartyProto.PartyNode commonPartyNode3 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(intersect.getName()) + .putAttributes("intersect_method",intersectMethod) + .putAttributes("sync_intersect_ids",syncIntersectIds) + .putAttributes("only_output_key",onlyOutputKey) + .putAttributes("rsa_params", + getCommonConfRsaParamTypedValueProto()) + .build(); + + List guestPartyNodes = + new ArrayList<>(); + guestPartyNodes.add(guestPartyNode1); + guestPartyNodes.add(guestPartyNode2); + guestPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto guest = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.GUEST).addAllNodes(guestPartyNodes).setName("9999").build(); + // guest 已经组装好 + + // 开始组装host + + // Map hostTableMap = + // new HashMap<>(); + // hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + // hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto hostMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + // HighflipMeta.ValueProto hostValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + // highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + + HighflipMeta.PartyProto.PartyNode hostPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getHostReaderTypedValueProto()) + .build(); + + // + HighflipMeta.PartyProto.PartyNode hostPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + List hostPartyNodes = + new ArrayList<>(); + hostPartyNodes.add(hostPartyNode1); + hostPartyNodes.add(hostPartyNode2); + hostPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto host = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.HOST).addAllNodes(hostPartyNodes).setName("10000").build(); + + List node = new ArrayList<>(); + node.add(reader); + node.add(dataTransform); + node.add(intersect); + + List partyProtoList = new ArrayList<>(); + partyProtoList.add(guest); + partyProtoList.add(host); + + HighflipMeta.GraphProto graphProto = + HighflipMeta.GraphProto.newBuilder().setName("jobId-202301") + .addAllNodes(node) + .addAllParties(partyProtoList).build(); + + return graphProto; + } + + private String getIntersectionInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("data_transform_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getDataTransformInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("reader_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfDataOutput() { + List data = new ArrayList<>(); + data.add("data"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfModelOutput() { + List data = new ArrayList<>(); + data.add("model"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private HighflipMeta.TypedValueProto getGuestPartyReaderNode() { + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + return typedValueProto; + } + + private HighflipMeta.TypedValueProto getCommonConfRsaParamTypedValueProto(){ + Map commonRsaParamConfMap = + new HashMap<>(); + commonRsaParamConfMap.put("hash_method", + HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("split_calculation", + HighflipMeta.ValueProto.newBuilder().setBool( + false).build()); + commonRsaParamConfMap.put("key_length", + HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + HighflipMeta.MapProto commonValueMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + HighflipMeta.ValueProto commonValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + return commonTypedValueProto; + } + + private HighflipMeta.TypedValueProto getHostReaderTypedValueProto() { + Map hostTableMap = + new HashMap<>(); + hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto hostMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + HighflipMeta.ValueProto hostValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + return hostTypedValueProto; + } + } diff --git a/highflip-doc/pom.xml b/highflip-doc/pom.xml index 0d8e57a..3ace508 100644 --- a/highflip-doc/pom.xml +++ b/highflip-doc/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-doc pom 1.0.0-SNAPSHOT diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml new file mode 100644 index 0000000..57aa44c --- /dev/null +++ b/highflip-editor/pom.xml @@ -0,0 +1,97 @@ + + + + highflip + com.baidu.highflip + 1.0.0-SNAPSHOT + + 4.0.0 + + highflip-editor + + + + com.baidu.highflip + highflip-sdk + ${project.version} + + + com.baidu.highflip + highflip-proto + ${project.version} + + + + org.springframework.boot + spring-boot-starter-web + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.springframework.boot + spring-boot-starter-aop + ${spring.boot.version} + + + ch.qos.logback + logback-classic + + + + + org.springframework.boot + spring-boot-autoconfigure + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + + org.springframework.security + spring-security-config + ${spring.security.version} + + + + + org.springframework.boot + spring-boot-starter-log4j + 1.3.8.RELEASE + + + + org.projectlombok + lombok + ${lombok.version} + + + org.testng + testng + ${testng.version} + test + + + org.jmockit + jmockit + ${jmockit.version} + test + + + com.google.guava + guava + ${guava.version} + + + + \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java new file mode 100644 index 0000000..007dc29 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java @@ -0,0 +1,34 @@ +package com.baidu.highflip.editor.model; + +public class HighFlipEditorException extends RuntimeException { + + private int code; + private String message; + + public HighFlipEditorException(int code, String message) { + super(message); + this.code = code; + this.message = message; + } + + public HighFlipEditorException(String message) { + super(message); + this.code = 400; + this.message = message; + } + + public HighFlipEditorException(int code, String message, Throwable cause) { + super(message, cause); + this.code = code; + this.message = message; + } + + public int getCode() { + return code; + } + + @Override + public String getMessage() { + return message; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java new file mode 100644 index 0000000..dba016a --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java @@ -0,0 +1,66 @@ +package com.baidu.highflip.editor.model; + +/** + * customized response of controller + */ +public class Result { + /** + * similar to http status code + */ + private int code; + private String message; + private T data; + + public Result() { + } + + public Result(int code, String message, T data) { + this.code = code; + this.message = message; + this.data = data; + } + + public static Result success() { + Result result = new Result(); + result.code = 200; + return result; + } + + public static Result success(T data) { + Result result = new Result(); + result.code = 200; + result.data = data; + return result; + } + + public static Result failure(int errorCode, String message) { + Result result = new Result(); + result.code = errorCode; + result.message = message; + return result; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public T getData() { + return data; + } + + public void setCode(int code) { + this.code = code; + } + + public void setMessage(String message) { + this.message = message; + } + + public void setData(T data) { + this.data = data; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java new file mode 100644 index 0000000..4e0fbc5 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java @@ -0,0 +1,50 @@ +package com.baidu.highflip.editor.utils; + + +import java.io.IOException; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * utils for serialization and deserialization + * + * @author songtingyu + */ +public class JacksonUtils { + + private static final ObjectMapper mapper = new ObjectMapper(); + + static { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + mapper.configure(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS, false); + } + + private JacksonUtils() { + + } + + public static String toJson(T obj) { + try { + return mapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + throw new HighFlipEditorException(400, "failed to serialize obj.", e); + } + } + + public static T fromJson(String json, Class valueType) { + if (json == null || json.isEmpty()) { + throw new HighFlipEditorException(400, "json is empty!"); + } + try { + return mapper.readValue(json, valueType); + } catch (IOException e) { + throw new HighFlipEditorException(400, e.getMessage(), e); + } + } +} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java new file mode 100644 index 0000000..e1ea1b1 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java @@ -0,0 +1,21 @@ +package com.baidu.highflip.editor.utils; + +public class StringUtils { + + public static boolean isNotBlank(final CharSequence cs) { + return !isBlank(cs); + } + + public static boolean isBlank(final CharSequence cs) { + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (Character.isWhitespace(cs.charAt(i)) == false) { + return false; + } + } + return true; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java new file mode 100644 index 0000000..40a49a9 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListDataRequest { + + String partnerId; + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java new file mode 100644 index 0000000..e813154 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java @@ -0,0 +1,16 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListOperatorsRequest { + + String highFlipServerIp; + + int highFlipServerPort; + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java new file mode 100644 index 0000000..a240873 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java @@ -0,0 +1,15 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListPartnersRequest { + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java new file mode 100644 index 0000000..c0cf92b --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java @@ -0,0 +1,19 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class LoginRequest { + + private String username; + + private String password; + + private String serverIp; + + private int serverPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java new file mode 100644 index 0000000..e25ec31 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SaveJobRequest { + + String json; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java new file mode 100644 index 0000000..f53568e --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java @@ -0,0 +1,39 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; + +/** + * global exception handler + */ +@ControllerAdvice +public class RestResponseEntityExceptionHandler + extends ResponseEntityExceptionHandler { + + @ExceptionHandler(value = {RuntimeException.class}) + protected ResponseEntity handleRuntimeException( + RuntimeException e, WebRequest request) { + Result result = Result.failure(400, e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.INTERNAL_SERVER_ERROR, + request); + } + + @ExceptionHandler(value = {HighFlipEditorException.class}) + protected ResponseEntity handleDsException( + HighFlipEditorException e, WebRequest request) { + Result result = Result.failure(e.getCode(), e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.OK, request); + } +} + + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java new file mode 100644 index 0000000..6d903d2 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java @@ -0,0 +1,11 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class WebApplication { + public static void main(String[] args) { + SpringApplication.run(WebApplication.class, args); + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java new file mode 100644 index 0000000..cc8f409 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.editor.web; + +public class WebSecurityConfig { + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java new file mode 100644 index 0000000..77c9a77 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.web.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * ignore request body when print access log. + * use in {@link com.baidu.highflip.editor.web.aop.AccessLogAop} + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface IgnoreRequestBody { +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java new file mode 100644 index 0000000..8179e89 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java @@ -0,0 +1,91 @@ +package com.baidu.highflip.editor.web.aop; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import com.baidu.highflip.editor.utils.JacksonUtils; +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + +@Aspect +@Component +@Order(0) +public class AccessLogAop extends RequestAspect { + + private static final String LOG_CONTENT = + "RequestMethod = {}, RequestUri = {}, RemoteAddress = {}, " + + "RequestParams = `{}`, RequestBody = {}"; + // when request method is post or put, log request body + private static final Set REQUEST_BODY_METHOD = + ImmutableSet.of(RequestMethod.POST.name(), + RequestMethod.PUT.name()); + + private Map loggerHolder = Maps.newConcurrentMap(); + + @Before(CONTROLLER_METHOD_NAME) + public void before(JoinPoint joinPoint) { + Class targetClass = joinPoint.getTarget().getClass(); + + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + Method method = signature.getMethod(); + + Object[] args = joinPoint.getArgs(); + + log(targetClass, method, args); + } + + private void log(Class targetClass, Method method, Object[] args) { + // 1. get logger + Logger logger = loggerHolder.putIfAbsent( + targetClass, LoggerFactory.getLogger(targetClass)); + logger = logger == null ? loggerHolder.get(targetClass) : logger; + + // 2. get request + ServletRequestAttributes requestAttributes = + (ServletRequestAttributes) RequestContextHolder + .currentRequestAttributes(); + HttpServletRequest request = requestAttributes.getRequest(); + + // 3. get log content + // request body is a method arg, that has @RequestBody annotation. + String remoteAddress = request.getRemoteAddr(); + String url = request.getRequestURI(); + String requestParams = request.getQueryString(); + Object body = null; + if (REQUEST_BODY_METHOD.stream() + .anyMatch(m -> m.equals(request.getMethod()))) { + if (method.getAnnotation(IgnoreRequestBody.class) == null) { + Annotation[][] annotations = method.getParameterAnnotations(); + for (int i = 0; i < annotations.length; i++) { + for (int j = 0; j < annotations[i].length; j++) { + if (annotations[i][j] instanceof RequestBody) { + body = args[i]; + } + } + } + } + } + + logger.info(LOG_CONTENT, + targetClass.getSimpleName() + "." + method.getName(), + url, remoteAddress, requestParams, JacksonUtils.toJson(body)); + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java new file mode 100644 index 0000000..85ab08f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.web.aop; + +import org.aspectj.lang.annotation.Pointcut; + +public class RequestAspect { + + protected static final String CONTROLLER_METHOD_NAME = "controllerMethod()"; + + @Pointcut("@annotation(org.springframework.web.bind.annotation.RequestMapping)") + protected void controllerMethod() { + + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java new file mode 100644 index 0000000..3300b95 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -0,0 +1,105 @@ +package com.baidu.highflip.editor.web.controller; + +import java.util.List; + +import javax.servlet.http.HttpServletRequest; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.utils.StringUtils; +import com.baidu.highflip.editor.vo.ListDataRequest; +import com.baidu.highflip.editor.vo.ListOperatorsRequest; +import com.baidu.highflip.editor.vo.ListPartnersRequest; +import com.baidu.highflip.editor.vo.SaveJobRequest; +import com.baidu.highflip.editor.web.services.HighFlipEditorService; +import com.baidu.highflip.editor.vo.LoginRequest; +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; + +import highflip.v1.Highflip; + +@RestController +@RequestMapping(value = "highflip") +public class HighFlipController { + + private static String TOKEN = "Token"; + + @Autowired + HighFlipEditorService highFlipEditorService; + + @IgnoreRequestBody + @RequestMapping(value = "login", method = RequestMethod.POST) + public Result login(@RequestBody LoginRequest request) { + highFlipEditorService.login(request.getUsername(), + request.getPassword(), + request.getServerIp(), + request.getServerPort()); + return Result.success("login successfully"); + } + + @RequestMapping(value = "operators", method = RequestMethod.GET) + public Result listOperators(ListOperatorsRequest request) { + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List operators = + highFlipEditorService.getOperators( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + + return Result.success(operators); + } + + @RequestMapping(value = "partners", method = RequestMethod.GET) + public Result listPartners(ListPartnersRequest request){ + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List partners = + highFlipEditorService.getPartners( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + return Result.success(partners); + } + + @RequestMapping(value = "data", method = RequestMethod.GET) + public Result listData(ListDataRequest request) { + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List dataGetResponseList = + highFlipEditorService.getDataList( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password, request.getPartnerId()); + return Result.success(dataGetResponseList); + } + + @RequestMapping(value = "job", method = RequestMethod.POST) + public Result saveJob(HttpServletRequest request, + @RequestBody SaveJobRequest saveJobRequest) { + String token = request.getHeader(TOKEN); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + highFlipEditorService.saveJob(token, saveJobRequest.getJson()); + return Result.success(); + } + + @RequestMapping(value = "job", method = RequestMethod.GET) + public Result getJob(HttpServletRequest request) { + String token = request.getHeader(TOKEN); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + String jobDag = highFlipEditorService.getJob(token); + return Result.success(jobDag); + } + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java new file mode 100644 index 0000000..57e0e7f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java @@ -0,0 +1,47 @@ +package com.baidu.highflip.editor.web.filter; + +import java.io.IOException; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; +import com.fasterxml.jackson.databind.ObjectMapper; + +@Component +public class GlobalExceptionFilter extends OncePerRequestFilter { + + @Autowired + protected ObjectMapper objectMapper; + + @Override + protected void doFilterInternal(HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain) + throws ServletException, IOException { + try { + filterChain.doFilter(request, response); + } catch (Exception e) { + response.setContentType("application/json;charset=UTF-8"); + // handle TicketException + if (e instanceof HighFlipEditorException) { + HighFlipEditorException ticketException = (HighFlipEditorException) e; + objectMapper.writeValue(response.getWriter(), + Result.failure( + ticketException.getCode(), + ticketException.getMessage())); + } else { + objectMapper.writeValue(response.getWriter(), + Result.failure(400, e.getMessage())); + } + } + } +} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java new file mode 100644 index 0000000..9e274dc --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -0,0 +1,113 @@ +package com.baidu.highflip.editor.web.services; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.baidu.highflip.client.HighFlipClient; +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.utils.StringUtils; + +import highflip.v1.Highflip; + +@Service +public class HighFlipEditorService { + + private static final Logger LOG = + LoggerFactory.getLogger(HighFlipEditorService.class); + + private ConcurrentHashMap jobs; + + public HighFlipEditorService() { + jobs = new ConcurrentHashMap<>(); + } + + public void login(String username, String password, + String serverIp, int serverPort) { + // TODO create grpc client to login + } + + public List getOperators(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo ="grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + // rpc call + Iterable operatorIds = highFlipClient.listOperators(0, 1000); + List operators = new ArrayList<>(); + for(String operatorId : operatorIds) { + Highflip.OperatorGetResponse operator = + highFlipClient.getOperator(operatorId); + operators.add(operator); + } + return operators; + } + + public List getPartners(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable partnerIds = highFlipClient.listPartners(0, 1000); + List partners = new ArrayList<>(); + for (String partnerId : partnerIds) { + Highflip.PartnerGetResponse partner = + highFlipClient.getPartner(partnerId); + partners.add(partner); + } + return partners; + } + + public List getDataList(String ip, + int port, + String username, + String password, + String targetPartnerId) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable dataIds = highFlipClient.listData(0, 1000); + List dataList = new ArrayList<>(); + for (String dataId : dataIds) { + Highflip.DataGetResponse data = highFlipClient.getData(dataId); + if (StringUtils.isBlank(targetPartnerId) || + (StringUtils.isNotBlank(data.getPartyId()) && + data.getPartyId().equals(targetPartnerId))) { + dataList.add(data); + } + } + return dataList; + } + + // save job + public void saveJob(String key, String jsonOfJob) { + if (jobs == null) { + jobs = new ConcurrentHashMap<>(); + } + jobs.put(key, jsonOfJob); + } + + public String getJob(String key){ + if(StringUtils.isBlank(key)){ + throw new HighFlipEditorException("Key is blank"); + } + return jobs.get(key); + } + +} diff --git a/highflip-editor/src/main/resources/application.properties b/highflip-editor/src/main/resources/application.properties new file mode 100644 index 0000000..b9cb39f --- /dev/null +++ b/highflip-editor/src/main/resources/application.properties @@ -0,0 +1,8 @@ +server.port=8090 + +server.servlet.session.timeout=30m +server.tomcat.connection-timeout=600000 +# httpOnly +server.servlet.session.cookie.http-only=true + +spring.web.resources.static-locations=classpath:/ui/ \ No newline at end of file diff --git a/highflip-editor/src/main/resources/log4j.properties b/highflip-editor/src/main/resources/log4j.properties new file mode 100644 index 0000000..ebec8d1 --- /dev/null +++ b/highflip-editor/src/main/resources/log4j.properties @@ -0,0 +1,14 @@ +# LOG4J configuration +log4j.rootLogger=info, console, file + +# Console Logger +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d [%p] %l %x - %m%n + +# File Logger +log4j.appender.file=org.apache.log4j.DailyRollingFileAppender +log4j.appender.file.File=./logs/highflip-editor.log +log4j.appender.file.DatePattern='-'yyyy-MM-dd'.log' +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d [%p] %l %x - %m%n \ No newline at end of file diff --git a/highflip-proto/pom.xml b/highflip-proto/pom.xml index 1584c1c..868c091 100644 --- a/highflip-proto/pom.xml +++ b/highflip-proto/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-proto jar 1.0.0-SNAPSHOT @@ -16,6 +16,19 @@ 1.49.0 + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + io.grpc diff --git a/highflip-proto/src/main/proto/highflip-meta.proto b/highflip-proto/src/main/proto/highflip-meta.proto index a093630..a11552c 100644 --- a/highflip-proto/src/main/proto/highflip-meta.proto +++ b/highflip-proto/src/main/proto/highflip-meta.proto @@ -65,6 +65,8 @@ message NodeInputProto{ string from_node = 4; string from_output = 5; + + string value = 6; } message NodeOutputProto{ @@ -74,6 +76,8 @@ message NodeOutputProto{ string name = 2; optional string description = 3; + + string value = 4; } message NodeProto{ diff --git a/highflip-proto/src/main/proto/highflip.proto b/highflip-proto/src/main/proto/highflip.proto index 880a5b6..119c543 100644 --- a/highflip-proto/src/main/proto/highflip.proto +++ b/highflip-proto/src/main/proto/highflip.proto @@ -230,6 +230,8 @@ message TaskGetResponse{ repeated string input_tasks = 9; repeated string output_tasks = 10; + + repeated string output_data_ids = 11; } @@ -559,6 +561,15 @@ message ConfigListResponse{ string key = 1; } +message GetServiceConfigResponse{ + + string url = 1; + + string party_id = 2; + + string role = 3; +} + //////////////////////////////////////////////////////////////////////////////// // SERVICE diff --git a/highflip-server/pom.xml b/highflip-server/pom.xml index 5055c3e..104b332 100644 --- a/highflip-server/pom.xml +++ b/highflip-server/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-server jar 1.0.0-SNAPSHOT @@ -11,7 +11,7 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml @@ -70,7 +70,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java index a9fdf38..e183cd5 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java @@ -2,10 +2,12 @@ import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.HighFlipAdaptor; +import com.baidu.highflip.core.engine.HighFlipRuntime; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.impl.ConfigurableOperatorAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePartnerAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePlatformAdaptor; +import com.baidu.highflip.server.adaptor.impl.ConfigurableServiceAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbJobAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbTaskAdaptor; import com.baidu.highflip.server.adaptor.impl.FixedSingleDataAdaptor; @@ -22,6 +24,17 @@ public class DefaultAdaptor implements HighFlipAdaptor { public static final String PLATFORM_PROPERTIES = "/adaptor/highflip.platform.properties"; + /** + * 用于配置本侧服务的配置信息 + */ + public static final String SERVICE_PROPERTIES = "/adaptor/highflip.service.properties"; + + private HighFlipRuntime highFlipRuntime; + + public DefaultAdaptor(HighFlipRuntime highFlipRuntime) { + this.highFlipRuntime = highFlipRuntime; + } + @Override public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_JOB, new DumbJobAdaptor()); @@ -37,6 +50,20 @@ public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_PLATFORM, new ConfigurablePlatformAdaptor(HighFlipUtils.getProperty(PLATFORM_PROPERTIES))); + + final ConfigurableServiceAdaptor configurableServiceAdaptor = + new ConfigurableServiceAdaptor( + HighFlipUtils.getProperty(SERVICE_PROPERTIES)); + register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, configurableServiceAdaptor); + + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.url", + configurableServiceAdaptor.getUrl()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.party.id", + configurableServiceAdaptor.getPartyId()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.role", + configurableServiceAdaptor.getRole()); + + register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } @Override diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java new file mode 100644 index 0000000..800bbc8 --- /dev/null +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java @@ -0,0 +1,42 @@ +package com.baidu.highflip.server.adaptor.impl; + +import java.util.Properties; + +import com.baidu.highflip.core.adaptor.ServiceAdaptor; +import com.baidu.highflip.core.common.AdaptorPropsList; + +public class ConfigurableServiceAdaptor implements ServiceAdaptor { + + String url; + + String partyId; + + String role; + + public ConfigurableServiceAdaptor(Properties props) { + + this.url = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT); + + this.partyId = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT); + + this.role = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT); + } + + @Override + public String getUrl() { + return this.url; + } + + @Override + public String getPartyId() { + return this.partyId; + } + + @Override + public String getRole() { + return this.role; + } +} diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java index 0c99b7e..2a7827d 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java @@ -3,6 +3,8 @@ import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.DefaultAdaptor; import com.baidu.highflip.server.adaptor.loader.AdaptorLoader; +import com.baidu.highflip.server.engine.component.HighFlipRuntime; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -23,6 +25,9 @@ public class AdaptorConfig { @Autowired InstanceRegister register; + @Autowired + HighFlipRuntime highFlipRuntime; + AdaptorLoader loader = null; @PostConstruct @@ -71,7 +76,7 @@ void initialAdaptor() { } void initialDefaultAdaptor() { - DefaultAdaptor adaptor = new DefaultAdaptor(); + DefaultAdaptor adaptor = new DefaultAdaptor(highFlipRuntime); adaptor.setup(register); } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java index 525db92..05443c3 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -64,7 +65,7 @@ public class HighFlipEngine { @Autowired PlatformTransactionManager transactionManager; - ConcurrentMap activeJobs; + ConcurrentMap activeJobs = new ConcurrentHashMap<>(); @Autowired AsyncTaskExecutor executor; @@ -290,7 +291,6 @@ public Job createJob(String name, String description, Graph graph) { job = getContext().getJobAdaptor() .createJob(job); - job = getContext().getJobRepository() .save(job); @@ -314,12 +314,12 @@ public Job createJob(String name, String description, Graph graph) { getContext().getTaskRepository() .saveAll(news); } + activeJobs.putIfAbsent(job.getJobId(), job); return job; } - // @Scheduled - protected void updateJob() { + public void updateJob() { JobAdaptor adaptor = getContext().getJobAdaptor(); @@ -328,6 +328,10 @@ protected void updateJob() { if (status != job.getStatus()) { job.setStatus(status); getContext().getJobRepository().save(job); + getContext().getJobAdaptor().updateJob(job); + } + if (status == Status.SUCCEEDED || status == Status.FAILED) { + activeJobs.remove(job.getJobId()); } }); } @@ -413,14 +417,15 @@ public void initializeTasks() { } // @Scheduled - private void updateTask() { - + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); } public Iterable listTask(String jobid) { - - return getContext().getTaskRepository() - .findAllByJobid(jobid); + final List tasks = getContext().getTaskRepository() + .findAllByJobid(jobid); + log.info("tasks size: {}", tasks.size()); + return tasks; } @Cacheable(value = "tasks") diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java index d204511..3d2a07b 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java @@ -5,6 +5,7 @@ import com.baidu.highflip.core.adaptor.OperatorAdaptor; import com.baidu.highflip.core.adaptor.PartnerAdaptor; import com.baidu.highflip.core.adaptor.PlatformAdaptor; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.adaptor.TaskAdaptor; import com.baidu.highflip.core.adaptor.UserAdaptor; import com.baidu.highflip.core.common.InstanceNameList; @@ -125,4 +126,9 @@ public UserAdaptor getUserAdaptor(){ public UserRepository getUserRepository() { return userReps; } + + // SERVICE + public ServiceAdaptor getServiceAdaptor() { + return getInstance(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java index 55c0412..791cce8 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java @@ -7,6 +7,8 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -18,13 +20,16 @@ public class HighFlipRuntime implements com.baidu.highflip.core.engine.HighFlipR @Autowired HighFlipContext context; + @Autowired + HighFlipConfiguration configuration; + HighFlipContext getContext() { return context; } @Override public Configuration getConfiguration() { - return null; + return configuration; } @Override @@ -69,4 +74,21 @@ public User getUser(String userId) { public Operator getOperator(String operatorId) { return null; } + + @Override + public Data registerData(Data data) { + Data dataSaved = getContext().getDataRepository().save(data); + return dataSaved; + } + + @Override + public Iterable listTask(String jobId) { + return getContext().getTaskRepository() + .findAllByJobid(jobId); + } + + @Override + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index aaa80e2..0640b8f 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -1,5 +1,6 @@ package com.baidu.highflip.server.rpc.v1; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.entity.dag.Graph; import com.baidu.highflip.core.entity.runtime.Config; import com.baidu.highflip.core.entity.runtime.Data; @@ -84,6 +85,7 @@ public void listConfig(Highflip.ConfigListRequest request, * @param request * @param responseObserver */ + @Override public void getConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -105,6 +107,7 @@ public void getConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void setConfig(Highflip.ConfigSetRequest request, StreamObserver responseObserver) { @@ -113,6 +116,7 @@ public void setConfig(Highflip.ConfigSetRequest request, returnVoid(responseObserver); } + @Override public void deleteConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -129,6 +133,7 @@ public void deleteConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void getPlatform(Highflip.Void request, StreamObserver responseObserver) { @@ -147,6 +152,7 @@ public void getPlatform(Highflip.Void request, * @param request * @param responseObserver */ + @Override public void matchPlatform(Highflip.PlatformMatchRequest request, StreamObserver responseObserver) { @@ -189,6 +195,7 @@ public void createJob(Highflip.JobCreateRequest request, * @param request * @param responseObserver */ + @Override public void getJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); @@ -210,16 +217,24 @@ public void getJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void checkJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); + com.baidu.highflip.core.entity.runtime.basic.Status jobStatus = + getEngine().getContext() + .getJobAdaptor() + .getJobStatus(job); + // check and update job + getEngine().updateJob(); Highflip.JobCheckResponse response = Highflip.JobCheckResponse .newBuilder() .setJobId(job.getJobId()) + .setStatus(Highflip.JobCheckResponse.JobStatus.valueOf( + jobStatus.toString().toUpperCase())) .build(); - returnOne(responseObserver, response); } @@ -227,6 +242,7 @@ public void checkJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void deleteJob(Highflip.JobId request, StreamObserver responseObserver) { @@ -239,6 +255,7 @@ public void deleteJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void listJob(Highflip.JobListRequest request, StreamObserver responseObserver) { @@ -259,6 +276,7 @@ public void listJob(Highflip.JobListRequest request, * @param request * @param responseObserver */ + @Override public void controlJob(Highflip.JobControlRequest request, StreamObserver responseObserver) { @@ -270,6 +288,7 @@ public void controlJob(Highflip.JobControlRequest request, returnVoid(responseObserver); } + @Override public void getJobLog(Highflip.JobLogRequest request, StreamObserver responseObserver) { @@ -282,11 +301,14 @@ public void getJobLog(Highflip.JobLogRequest request, * @param request * @param responseObserver */ + @Override public void listTask(Highflip.TaskListRequest request, StreamObserver responseObserver) { - + final Job job = getEngine().getJob(request.getJobId()); + log.info("job id: {}, job binging id: {}", job.getJobId(), + job.getBingingId()); Iterator response = Streams - .stream(getEngine().listTask(request.getJobId())) + .stream(getEngine().listTask(job.getBingingId())) .map(t -> Highflip.TaskListResponse .newBuilder() .setTaskId(t.getTaskid()) @@ -300,6 +322,7 @@ public void listTask(Highflip.TaskListRequest request, * @param request * @param responseObserver */ + @Override public void getTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -309,8 +332,10 @@ public void getTask(Highflip.TaskId request, .newBuilder() .setTaskId(task.getTaskid()) .setJobId(task.getJobid()) + .setNodeName(task.getNodeName()) .setCreateTime(task.getCreateTime().toString()) .setUpdateTime(task.getUpdateTime().toString()) + .addAllOutputDataIds(task.getOutputData()) .build(); returnOne(responseObserver, response); @@ -320,6 +345,7 @@ public void getTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void checkTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -340,6 +366,7 @@ public void checkTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void controlTask(Highflip.TaskControlRequest request, StreamObserver responseObserver) { @@ -355,6 +382,7 @@ public void controlTask(Highflip.TaskControlRequest request, * @param request * @param responseObserver */ + @Override public void getTaskLog(Highflip.TaskLogRequest request, StreamObserver responseObserver) { @@ -368,6 +396,7 @@ public void getTaskLog(Highflip.TaskLogRequest request, * @param request * @param responseObserver */ + @Override public void listData(Highflip.DataListRequest request, StreamObserver responseObserver) { @@ -383,6 +412,7 @@ public void listData(Highflip.DataListRequest request, * @param request * @param responseObserver */ + @Override public void getData(Highflip.DataId request, StreamObserver responseObserver) { @@ -413,6 +443,7 @@ public void getData(Highflip.DataId request, * @param responseObserver * @return */ + @Override public StreamObserver pushData( StreamObserver responseObserver) { @@ -496,6 +527,7 @@ public void onCompleted() { * @param request * @param responseObserver */ + @Override public void pullData(Highflip.DataPullRequest request, StreamObserver responseObserver) { @@ -537,6 +569,7 @@ public void pullData(Highflip.DataPullRequest request, * @param request * @param responseObserver */ + @Override public void deleteData(Highflip.DataId request, StreamObserver responseObserver) { @@ -552,6 +585,7 @@ public void deleteData(Highflip.DataId request, * @param request * @param responseObserver */ + @Override public void listOperator(Highflip.OperatorListRequest request, StreamObserver responseObserver) { Iterator response = Streams @@ -569,6 +603,7 @@ public void listOperator(Highflip.OperatorListRequest request, * @param request * @param responseObserver */ + @Override public void getOperator(Highflip.OperatorId request, StreamObserver responseObserver) { @@ -594,6 +629,7 @@ public void getOperator(Highflip.OperatorId request, * @param request * @param responseObserver */ + @Override public void createPartner(Highflip.PartnerCreateRequest request, StreamObserver responseObserver) { @@ -614,6 +650,7 @@ public void createPartner(Highflip.PartnerCreateRequest request, * @param request * @param responseObserver */ + @Override public void getPartner(Highflip.PartnerId request, StreamObserver responseObserver) { @@ -635,6 +672,7 @@ public void getPartner(Highflip.PartnerId request, * @param request * @param responseObserver */ + @Override public void listPartner(Highflip.PartnerListRequest request, StreamObserver responseObserver) { @@ -653,9 +691,9 @@ public void listPartner(Highflip.PartnerListRequest request, * @param request * @param responseObserver */ + @Override public void controlPartner(Highflip.PartnerControlRequest request, StreamObserver responseObserver) { } - } diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties new file mode 100644 index 0000000..b36b940 --- /dev/null +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -0,0 +1,3 @@ +highflip.adaptor.service.url=http://10.27.130.41:8380 +highflip.adaptor.service.party.id=9999 +highflip.adaptor.service.role=guest \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index e7c6ac5..48ed02f 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path= +highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar ######################################################### # spring grpc ######################################################### @@ -30,7 +30,9 @@ spring.jpa.properties.jadira.usertype.autoRegisterUserTypes=true # h2 spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect spring.datasource.driver-class-name=org.h2.Driver -spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip.db +spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip +spring.datasource.username=highflip +spring.datasource.password=changeMe ######################################################### # logging ######################################################### diff --git a/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java new file mode 100644 index 0000000..d4651d8 --- /dev/null +++ b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.server.engine; + +public class HighFlipEngineTest { + +} diff --git a/highflip-vendors/highflip-adaptor-demo/pom.xml b/highflip-vendors/highflip-adaptor-demo/pom.xml index a022666..871c819 100644 --- a/highflip-vendors/highflip-adaptor-demo/pom.xml +++ b/highflip-vendors/highflip-adaptor-demo/pom.xml @@ -3,7 +3,7 @@ 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-demo jar 1.0.0-SNAPSHOT @@ -18,7 +18,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT compile diff --git a/highflip-vendors/highflip-adaptor-fate/pom.xml b/highflip-vendors/highflip-adaptor-fate/pom.xml index 481c757..3fbcc3d 100644 --- a/highflip-vendors/highflip-adaptor-fate/pom.xml +++ b/highflip-vendors/highflip-adaptor-fate/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-fate jar 1.0.0-SNAPSHOT @@ -30,7 +30,7 @@ test - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT compile diff --git a/highflip-vendors/pom.xml b/highflip-vendors/pom.xml index 35c88ca..38a435b 100644 --- a/highflip-vendors/pom.xml +++ b/highflip-vendors/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-vendors pom 1.0.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 2d5fe7b..5358d7a 100644 --- a/pom.xml +++ b/pom.xml @@ -2,13 +2,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip pom 1.0.0-SNAPSHOT highflip - - http://maven.apache.org + High layer Federated Learning Intercommunication Protocol + https://github.com/baidu/highflip @@ -29,6 +29,17 @@ 11 11 1.49.0 + 2.7.3 + 1.18.22 + 6.11 + 1.43 + 31.1-jre + 6.0.2 + + + 3.2.0 + 1.6 + 1.6.7 @@ -38,8 +49,26 @@ highflip-server highflip-doc highflip-clients + highflip-editor + + + Baidu dianshi + dianshi-help@baidu.com + https://dianshi.baidu.com/ + dianshi + https://dianshi.baidu.com/ + + + + + scm:git:${project.scm.url} + scm:git:${project.scm.url} + https://github.com/baidu/highflip.git + HEAD + + @@ -119,6 +148,55 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadoc + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-source + + jar + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + + report + test + + report + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.plugin.version} + @@ -129,23 +207,114 @@ org.jacoco jacoco-maven-plugin - 0.8.2 - - - - prepare-agent - - - - - report - test - - report - - - + + + org.apache.maven.plugins + maven-jar-plugin + + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + + + + local + + true + + + + + release + + + + maven-source-plugin + + + create-source-jar + + jar + test-jar + + + + + + maven-jar-plugin + + + + test-jar + + + + true + + + + + + maven-javadoc-plugin + + + create-javadoc-jar + + javadoc + jar + + package + + + + ${java.version} + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven.gpg.plugin.version} + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${nexus.staging.maven.plugin.version} + true + + ossrh + https://oss.sonatype.org + true + + + + + + \ No newline at end of file