From 337303f2e256d89ebd3e0aa235afc81c8b12f551 Mon Sep 17 00:00:00 2001 From: songtingyu Date: Tue, 11 Apr 2023 16:47:21 +0800 Subject: [PATCH 01/17] init highflip editor module --- highflip-editor/pom.xml | 88 +++++++++++++++++++ .../editor/model/HighFlipEditorException.java | 34 +++++++ .../baidu/highflip/editor/model/Result.java | 66 ++++++++++++++ .../services/HighFlipEditorService.java | 20 +++++ .../highflip/editor/utils/JacksonUtils.java | 50 +++++++++++ .../highflip/editor/vo/LoginRequest.java | 19 ++++ .../RestResponseEntityExceptionHandler.java | 39 ++++++++ .../highflip/editor/web/WebApplication.java | 11 +++ .../editor/web/WebSecurityConfig.java | 5 ++ .../web/controller/ComponentController.java | 28 ++++++ .../web/filter/GlobalExceptionFilter.java | 47 ++++++++++ .../src/main/resources/application.properties | 6 ++ .../src/main/resources/log4j.properties | 14 +++ pom.xml | 1 + 14 files changed, 428 insertions(+) create mode 100644 highflip-editor/pom.xml create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java create mode 100644 highflip-editor/src/main/resources/application.properties create mode 100644 highflip-editor/src/main/resources/log4j.properties diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml new file mode 100644 index 0000000..a765a8a --- /dev/null +++ b/highflip-editor/pom.xml @@ -0,0 +1,88 @@ + + + + highflip + com.baidu + 1.0.0-SNAPSHOT + + 4.0.0 + + highflip-editor + + + 11 + 11 + 2.7.10 + 1.18.22 + 6.11 + 1.43 + + + + + + + + + + + org.springframework.boot + spring-boot-starter-web + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-logging + + + + + org.springframework.boot + spring-boot-autoconfigure + ${spring.boot.version} + + + org.springframework.boot + spring-boot-starter-test + ${spring.boot.version} + test + + + + org.springframework.security + spring-security-config + 6.0.2 + + + + + org.springframework.boot + spring-boot-starter-log4j + 1.3.8.RELEASE + + + + org.projectlombok + lombok + ${lombok.version} + + + org.testng + testng + ${testng.version} + test + + + org.jmockit + jmockit + ${jmockit.version} + test + + + + + + + \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java new file mode 100644 index 0000000..007dc29 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/HighFlipEditorException.java @@ -0,0 +1,34 @@ +package com.baidu.highflip.editor.model; + +public class HighFlipEditorException extends RuntimeException { + + private int code; + private String message; + + public HighFlipEditorException(int code, String message) { + super(message); + this.code = code; + this.message = message; + } + + public HighFlipEditorException(String message) { + super(message); + this.code = 400; + this.message = message; + } + + public HighFlipEditorException(int code, String message, Throwable cause) { + super(message, cause); + this.code = code; + this.message = message; + } + + public int getCode() { + return code; + } + + @Override + public String getMessage() { + return message; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java new file mode 100644 index 0000000..aa25331 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java @@ -0,0 +1,66 @@ +package com.baidu.highflip.editor.model; + +/** + * customized response of controller + */ +public class Result { + /** + * similar to http status code + */ + private int code; + private String message; + private Object data; + + public Result() { + } + + public Result(int code, String message, Object data) { + this.code = code; + this.message = message; + this.data = data; + } + + public static Result success() { + Result result = new Result(); + result.code = 200; + return result; + } + + public static Result success(Object data) { + Result result = new Result(); + result.code = 200; + result.data = data; + return result; + } + + public static Result failure(int errorCode, String message) { + Result result = new Result(); + result.code = errorCode; + result.message = message; + return result; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public Object getData() { + return data; + } + + public void setCode(int code) { + this.code = code; + } + + public void setMessage(String message) { + this.message = message; + } + + public void setData(Object data) { + this.data = data; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java new file mode 100644 index 0000000..15fd9ef --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java @@ -0,0 +1,20 @@ +package com.baidu.highflip.editor.services; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class HighFlipEditorService { + + private static final Logger LOG = + LoggerFactory.getLogger(HighFlipEditorService.class); + + public HighFlipEditorService() { + + } + + public void login(String username, String password, + String serverIp, int serverPort) { + // TODO create grpc client to login + } + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java new file mode 100644 index 0000000..4e0fbc5 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/JacksonUtils.java @@ -0,0 +1,50 @@ +package com.baidu.highflip.editor.utils; + + +import java.io.IOException; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * utils for serialization and deserialization + * + * @author songtingyu + */ +public class JacksonUtils { + + private static final ObjectMapper mapper = new ObjectMapper(); + + static { + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, + false); + mapper.configure(MapperFeature.ALLOW_FINAL_FIELDS_AS_MUTATORS, false); + } + + private JacksonUtils() { + + } + + public static String toJson(T obj) { + try { + return mapper.writeValueAsString(obj); + } catch (JsonProcessingException e) { + throw new HighFlipEditorException(400, "failed to serialize obj.", e); + } + } + + public static T fromJson(String json, Class valueType) { + if (json == null || json.isEmpty()) { + throw new HighFlipEditorException(400, "json is empty!"); + } + try { + return mapper.readValue(json, valueType); + } catch (IOException e) { + throw new HighFlipEditorException(400, e.getMessage(), e); + } + } +} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java new file mode 100644 index 0000000..c0cf92b --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/LoginRequest.java @@ -0,0 +1,19 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class LoginRequest { + + private String username; + + private String password; + + private String serverIp; + + private int serverPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java new file mode 100644 index 0000000..f53568e --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/RestResponseEntityExceptionHandler.java @@ -0,0 +1,39 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; + +/** + * global exception handler + */ +@ControllerAdvice +public class RestResponseEntityExceptionHandler + extends ResponseEntityExceptionHandler { + + @ExceptionHandler(value = {RuntimeException.class}) + protected ResponseEntity handleRuntimeException( + RuntimeException e, WebRequest request) { + Result result = Result.failure(400, e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.INTERNAL_SERVER_ERROR, + request); + } + + @ExceptionHandler(value = {HighFlipEditorException.class}) + protected ResponseEntity handleDsException( + HighFlipEditorException e, WebRequest request) { + Result result = Result.failure(e.getCode(), e.getMessage()); + return handleExceptionInternal(e, result, new HttpHeaders(), + HttpStatus.OK, request); + } +} + + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java new file mode 100644 index 0000000..6d903d2 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebApplication.java @@ -0,0 +1,11 @@ +package com.baidu.highflip.editor.web; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class WebApplication { + public static void main(String[] args) { + SpringApplication.run(WebApplication.class, args); + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java new file mode 100644 index 0000000..cc8f409 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/WebSecurityConfig.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.editor.web; + +public class WebSecurityConfig { + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java new file mode 100644 index 0000000..4d60b9f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java @@ -0,0 +1,28 @@ +package com.baidu.highflip.editor.web.controller; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.services.HighFlipEditorService; +import com.baidu.highflip.editor.vo.LoginRequest; + + +@RestController +public class ComponentController { + + @RequestMapping(value = "/login", method = RequestMethod.POST) + public Result login(@RequestBody LoginRequest request) { + HighFlipEditorService highFlipEditorService = + new HighFlipEditorService(); + highFlipEditorService.login(request.getUsername(), + request.getPassword(), + request.getServerIp(), + request.getServerPort()); + return Result.success(); + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java new file mode 100644 index 0000000..57e0e7f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/filter/GlobalExceptionFilter.java @@ -0,0 +1,47 @@ +package com.baidu.highflip.editor.web.filter; + +import java.io.IOException; + +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import com.baidu.highflip.editor.model.HighFlipEditorException; +import com.baidu.highflip.editor.model.Result; +import com.fasterxml.jackson.databind.ObjectMapper; + +@Component +public class GlobalExceptionFilter extends OncePerRequestFilter { + + @Autowired + protected ObjectMapper objectMapper; + + @Override + protected void doFilterInternal(HttpServletRequest request, + HttpServletResponse response, + FilterChain filterChain) + throws ServletException, IOException { + try { + filterChain.doFilter(request, response); + } catch (Exception e) { + response.setContentType("application/json;charset=UTF-8"); + // handle TicketException + if (e instanceof HighFlipEditorException) { + HighFlipEditorException ticketException = (HighFlipEditorException) e; + objectMapper.writeValue(response.getWriter(), + Result.failure( + ticketException.getCode(), + ticketException.getMessage())); + } else { + objectMapper.writeValue(response.getWriter(), + Result.failure(400, e.getMessage())); + } + } + } +} + diff --git a/highflip-editor/src/main/resources/application.properties b/highflip-editor/src/main/resources/application.properties new file mode 100644 index 0000000..2d3e216 --- /dev/null +++ b/highflip-editor/src/main/resources/application.properties @@ -0,0 +1,6 @@ +server.port=8090 + +server.servlet.session.timeout=30m +server.tomcat.connection-timeout=600000 +# httpOnly +server.servlet.session.cookie.http-only=true diff --git a/highflip-editor/src/main/resources/log4j.properties b/highflip-editor/src/main/resources/log4j.properties new file mode 100644 index 0000000..63e913e --- /dev/null +++ b/highflip-editor/src/main/resources/log4j.properties @@ -0,0 +1,14 @@ +# LOG4J configuration +log4j.rootLogger=info, console, file + +# Console Logger +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d [%p] %l %x - %m%n + +# File Logger +log4j.appender.file=org.apache.log4j.DailyRollingFileAppender +log4j.appender.file.File=./logs/ds-agent.log +log4j.appender.file.DatePattern='-'yyyy-MM-dd'.log' +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.layout.ConversionPattern=%d [%p] %l %x - %m%n \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2d5fe7b..eb41ecc 100644 --- a/pom.xml +++ b/pom.xml @@ -38,6 +38,7 @@ highflip-server highflip-doc highflip-clients + highflip-editor From e4d55eec44b0fcdefb5844d34700d1b2838e6d73 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Sun, 23 Apr 2023 16:17:41 +0800 Subject: [PATCH 02/17] add access log aop of web api --- highflip-editor/pom.xml | 17 ++++ .../web/annotation/IgnoreRequestBody.java | 17 ++++ .../highflip/editor/web/aop/AccessLogAop.java | 91 +++++++++++++++++++ .../editor/web/aop/RequestAspect.java | 13 +++ .../web/controller/ComponentController.java | 7 +- .../src/main/resources/application.properties | 2 + 6 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index a765a8a..b81796e 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -18,6 +18,7 @@ 1.18.22 6.11 1.43 + 31.1-jre @@ -38,6 +39,17 @@ + + org.springframework.boot + spring-boot-starter-aop + ${spring.boot.version} + + + ch.qos.logback + logback-classic + + + org.springframework.boot spring-boot-autoconfigure @@ -80,6 +92,11 @@ ${jmockit.version} test + + com.google.guava + guava + ${guava.version} + diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java new file mode 100644 index 0000000..880ddeb --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.web.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * ignore request body when print access log. + * use in {@link com.baidu.dragonshare.web.aop.AccessLogAop} + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface IgnoreRequestBody { +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java new file mode 100644 index 0000000..8179e89 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/AccessLogAop.java @@ -0,0 +1,91 @@ +package com.baidu.highflip.editor.web.aop; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.Set; + +import javax.servlet.http.HttpServletRequest; + +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; + +import com.baidu.highflip.editor.utils.JacksonUtils; +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; + +@Aspect +@Component +@Order(0) +public class AccessLogAop extends RequestAspect { + + private static final String LOG_CONTENT = + "RequestMethod = {}, RequestUri = {}, RemoteAddress = {}, " + + "RequestParams = `{}`, RequestBody = {}"; + // when request method is post or put, log request body + private static final Set REQUEST_BODY_METHOD = + ImmutableSet.of(RequestMethod.POST.name(), + RequestMethod.PUT.name()); + + private Map loggerHolder = Maps.newConcurrentMap(); + + @Before(CONTROLLER_METHOD_NAME) + public void before(JoinPoint joinPoint) { + Class targetClass = joinPoint.getTarget().getClass(); + + MethodSignature signature = (MethodSignature) joinPoint.getSignature(); + Method method = signature.getMethod(); + + Object[] args = joinPoint.getArgs(); + + log(targetClass, method, args); + } + + private void log(Class targetClass, Method method, Object[] args) { + // 1. get logger + Logger logger = loggerHolder.putIfAbsent( + targetClass, LoggerFactory.getLogger(targetClass)); + logger = logger == null ? loggerHolder.get(targetClass) : logger; + + // 2. get request + ServletRequestAttributes requestAttributes = + (ServletRequestAttributes) RequestContextHolder + .currentRequestAttributes(); + HttpServletRequest request = requestAttributes.getRequest(); + + // 3. get log content + // request body is a method arg, that has @RequestBody annotation. + String remoteAddress = request.getRemoteAddr(); + String url = request.getRequestURI(); + String requestParams = request.getQueryString(); + Object body = null; + if (REQUEST_BODY_METHOD.stream() + .anyMatch(m -> m.equals(request.getMethod()))) { + if (method.getAnnotation(IgnoreRequestBody.class) == null) { + Annotation[][] annotations = method.getParameterAnnotations(); + for (int i = 0; i < annotations.length; i++) { + for (int j = 0; j < annotations[i].length; j++) { + if (annotations[i][j] instanceof RequestBody) { + body = args[i]; + } + } + } + } + } + + logger.info(LOG_CONTENT, + targetClass.getSimpleName() + "." + method.getName(), + url, remoteAddress, requestParams, JacksonUtils.toJson(body)); + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java new file mode 100644 index 0000000..85ab08f --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/aop/RequestAspect.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.web.aop; + +import org.aspectj.lang.annotation.Pointcut; + +public class RequestAspect { + + protected static final String CONTROLLER_METHOD_NAME = "controllerMethod()"; + + @Pointcut("@annotation(org.springframework.web.bind.annotation.RequestMapping)") + protected void controllerMethod() { + + } +} \ No newline at end of file diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java index 4d60b9f..ac1921a 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java @@ -1,7 +1,5 @@ package com.baidu.highflip.editor.web.controller; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; @@ -10,11 +8,12 @@ import com.baidu.highflip.editor.model.Result; import com.baidu.highflip.editor.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; - +import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; @RestController public class ComponentController { + @IgnoreRequestBody @RequestMapping(value = "/login", method = RequestMethod.POST) public Result login(@RequestBody LoginRequest request) { HighFlipEditorService highFlipEditorService = @@ -23,6 +22,6 @@ public Result login(@RequestBody LoginRequest request) { request.getPassword(), request.getServerIp(), request.getServerPort()); - return Result.success(); + return Result.success("login successfully"); } } diff --git a/highflip-editor/src/main/resources/application.properties b/highflip-editor/src/main/resources/application.properties index 2d3e216..b9cb39f 100644 --- a/highflip-editor/src/main/resources/application.properties +++ b/highflip-editor/src/main/resources/application.properties @@ -4,3 +4,5 @@ server.servlet.session.timeout=30m server.tomcat.connection-timeout=600000 # httpOnly server.servlet.session.cookie.http-only=true + +spring.web.resources.static-locations=classpath:/ui/ \ No newline at end of file From 155d79698fd7bef015bd2529d1286aa02bc822fe Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Mon, 24 Apr 2023 16:26:18 +0800 Subject: [PATCH 03/17] list operators --- highflip-clients/highflip-sdk/pom.xml | 2 +- .../baidu/highflip/client/HighFlipClient.java | 6 +++ highflip-editor/pom.xml | 19 ++++--- .../baidu/highflip/editor/model/Result.java | 14 +++--- .../services/HighFlipEditorService.java | 20 -------- .../editor/vo/ListOperatorsRequest.java | 16 ++++++ .../web/annotation/IgnoreRequestBody.java | 2 +- ...ontroller.java => OperatorController.java} | 30 +++++++++-- .../web/services/HighFlipEditorService.java | 50 +++++++++++++++++++ .../src/main/resources/log4j.properties | 2 +- 10 files changed, 120 insertions(+), 41 deletions(-) delete mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java rename highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/{ComponentController.java => OperatorController.java} (50%) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java diff --git a/highflip-clients/highflip-sdk/pom.xml b/highflip-clients/highflip-sdk/pom.xml index d982e95..b44148a 100644 --- a/highflip-clients/highflip-sdk/pom.xml +++ b/highflip-clients/highflip-sdk/pom.xml @@ -20,7 +20,7 @@ com.baidu highflip-proto - 1.0.0-SNAPSHOT + ${project.version} org.projectlombok diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 9a57d8c..8a68fd9 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -42,6 +42,12 @@ public HighFlipClient() { this.blockingStub = null; } + public static HighFlipClient newHighFlipClient(String target) { + HighFlipClient highFlipClient = new HighFlipClient(); + highFlipClient.connect(target); + return highFlipClient; + } + public void connect(String target) { close(); diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index b81796e..7f2e216 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -19,14 +19,20 @@ 6.11 1.43 31.1-jre + 6.0.2 - - - - - + + com.baidu + highflip-sdk + ${project.version} + + + com.baidu + highflip-proto + ${project.version} + org.springframework.boot @@ -65,7 +71,7 @@ org.springframework.security spring-security-config - 6.0.2 + ${spring.security.version} @@ -97,7 +103,6 @@ guava ${guava.version} - diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java index aa25331..dba016a 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/model/Result.java @@ -3,18 +3,18 @@ /** * customized response of controller */ -public class Result { +public class Result { /** * similar to http status code */ private int code; private String message; - private Object data; + private T data; public Result() { } - public Result(int code, String message, Object data) { + public Result(int code, String message, T data) { this.code = code; this.message = message; this.data = data; @@ -26,8 +26,8 @@ public static Result success() { return result; } - public static Result success(Object data) { - Result result = new Result(); + public static Result success(T data) { + Result result = new Result(); result.code = 200; result.data = data; return result; @@ -48,7 +48,7 @@ public String getMessage() { return message; } - public Object getData() { + public T getData() { return data; } @@ -60,7 +60,7 @@ public void setMessage(String message) { this.message = message; } - public void setData(Object data) { + public void setData(T data) { this.data = data; } } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java deleted file mode 100644 index 15fd9ef..0000000 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/services/HighFlipEditorService.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.baidu.highflip.editor.services; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class HighFlipEditorService { - - private static final Logger LOG = - LoggerFactory.getLogger(HighFlipEditorService.class); - - public HighFlipEditorService() { - - } - - public void login(String username, String password, - String serverIp, int serverPort) { - // TODO create grpc client to login - } - -} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java new file mode 100644 index 0000000..e813154 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListOperatorsRequest.java @@ -0,0 +1,16 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListOperatorsRequest { + + String highFlipServerIp; + + int highFlipServerPort; + +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java index 880ddeb..77c9a77 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/annotation/IgnoreRequestBody.java @@ -8,7 +8,7 @@ /** * ignore request body when print access log. - * use in {@link com.baidu.dragonshare.web.aop.AccessLogAop} + * use in {@link com.baidu.highflip.editor.web.aop.AccessLogAop} */ @Target(ElementType.METHOD) @Retention(RetentionPolicy.RUNTIME) diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java similarity index 50% rename from highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java rename to highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java index ac1921a..3744f2e 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/ComponentController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java @@ -1,27 +1,49 @@ package com.baidu.highflip.editor.web.controller; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import com.baidu.highflip.editor.model.Result; -import com.baidu.highflip.editor.services.HighFlipEditorService; +import com.baidu.highflip.editor.vo.ListOperatorsRequest; +import com.baidu.highflip.editor.web.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; +import highflip.v1.Highflip; + @RestController -public class ComponentController { +public class OperatorController { + + @Autowired + HighFlipEditorService highFlipEditorService; @IgnoreRequestBody @RequestMapping(value = "/login", method = RequestMethod.POST) public Result login(@RequestBody LoginRequest request) { - HighFlipEditorService highFlipEditorService = - new HighFlipEditorService(); highFlipEditorService.login(request.getUsername(), request.getPassword(), request.getServerIp(), request.getServerPort()); return Result.success("login successfully"); } + + @RequestMapping(method = RequestMethod.GET) + public Result listOperators(ListOperatorsRequest request) { + + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List operators = + highFlipEditorService.getOperators( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + + return Result.success(operators); + } } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java new file mode 100644 index 0000000..3ac9764 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -0,0 +1,50 @@ +package com.baidu.highflip.editor.web.services; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import com.baidu.highflip.client.HighFlipClient; + +import highflip.v1.Highflip; + +@Service +public class HighFlipEditorService { + + private static final Logger LOG = + LoggerFactory.getLogger(HighFlipEditorService.class); + + public HighFlipEditorService() { + + } + + public void login(String username, String password, + String serverIp, int serverPort) { + // TODO create grpc client to login + } + + + public List getOperators(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo ="grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + // rpc call + Iterable operatorIds = highFlipClient.listOperators(0, 1000); + List operators = new ArrayList<>(); + for(String operatorId : operatorIds) { + Highflip.OperatorGetResponse operator = + highFlipClient.getOperator(operatorId); + operators.add(operator); + } + return operators; + } + +} diff --git a/highflip-editor/src/main/resources/log4j.properties b/highflip-editor/src/main/resources/log4j.properties index 63e913e..ebec8d1 100644 --- a/highflip-editor/src/main/resources/log4j.properties +++ b/highflip-editor/src/main/resources/log4j.properties @@ -8,7 +8,7 @@ log4j.appender.console.layout.ConversionPattern=%d [%p] %l %x - %m%n # File Logger log4j.appender.file=org.apache.log4j.DailyRollingFileAppender -log4j.appender.file.File=./logs/ds-agent.log +log4j.appender.file.File=./logs/highflip-editor.log log4j.appender.file.DatePattern='-'yyyy-MM-dd'.log' log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout.ConversionPattern=%d [%p] %l %x - %m%n \ No newline at end of file From 1a43e7398b7011e3cd1a71d1b379f751e72eb7b9 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Tue, 25 Apr 2023 11:24:33 +0800 Subject: [PATCH 04/17] add web api of list partners,data --- .../highflip/editor/utils/StringUtils.java | 21 +++++++++ .../highflip/editor/vo/ListDataRequest.java | 17 +++++++ .../editor/vo/ListPartnersRequest.java | 15 ++++++ ...ontroller.java => HighFlipController.java} | 33 +++++++++++-- .../web/services/HighFlipEditorService.java | 46 ++++++++++++++++++- .../server/rpc/v1/HighFlipRpcService.java | 27 +++++++++++ 6 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java rename highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/{OperatorController.java => HighFlipController.java} (55%) diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java new file mode 100644 index 0000000..e1ea1b1 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/utils/StringUtils.java @@ -0,0 +1,21 @@ +package com.baidu.highflip.editor.utils; + +public class StringUtils { + + public static boolean isNotBlank(final CharSequence cs) { + return !isBlank(cs); + } + + public static boolean isBlank(final CharSequence cs) { + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (Character.isWhitespace(cs.charAt(i)) == false) { + return false; + } + } + return true; + } +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java new file mode 100644 index 0000000..40a49a9 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListDataRequest.java @@ -0,0 +1,17 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListDataRequest { + + String partnerId; + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java new file mode 100644 index 0000000..a240873 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/ListPartnersRequest.java @@ -0,0 +1,15 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ListPartnersRequest { + + String highFlipServerIp; + + int highFlipServerPort; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java similarity index 55% rename from highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java rename to highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java index 3744f2e..7a7b0c3 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/OperatorController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -9,7 +9,9 @@ import org.springframework.web.bind.annotation.RestController; import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.vo.ListDataRequest; import com.baidu.highflip.editor.vo.ListOperatorsRequest; +import com.baidu.highflip.editor.vo.ListPartnersRequest; import com.baidu.highflip.editor.web.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; @@ -17,7 +19,7 @@ import highflip.v1.Highflip; @RestController -public class OperatorController { +public class HighFlipController { @Autowired HighFlipEditorService highFlipEditorService; @@ -32,9 +34,8 @@ public Result login(@RequestBody LoginRequest request) { return Result.success("login successfully"); } - @RequestMapping(method = RequestMethod.GET) + @RequestMapping(value = "operators", method = RequestMethod.GET) public Result listOperators(ListOperatorsRequest request) { - // TODO: how to transfer username and password String username = "username"; String password = "password"; @@ -46,4 +47,30 @@ public Result listOperators(ListOperatorsRequest request) { return Result.success(operators); } + + @RequestMapping(value = "partners", method = RequestMethod.GET) + public Result listPartners(ListPartnersRequest request){ + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List partners = + highFlipEditorService.getPartners( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password); + return Result.success(partners); + } + + @RequestMapping(value = "data", method = RequestMethod.GET) + public Result listData(ListDataRequest request) { + // TODO: how to transfer username and password + String username = "username"; + String password = "password"; + List dataGetResponseList = + highFlipEditorService.getDataList( + request.getHighFlipServerIp(), + request.getHighFlipServerPort(), + username, password, request.getPartnerId()); + return Result.success(dataGetResponseList); + } } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java index 3ac9764..5a9e4fd 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -8,6 +8,7 @@ import org.springframework.stereotype.Service; import com.baidu.highflip.client.HighFlipClient; +import com.baidu.highflip.editor.utils.StringUtils; import highflip.v1.Highflip; @@ -26,7 +27,6 @@ public void login(String username, String password, // TODO create grpc client to login } - public List getOperators(String ip, int port, String username, @@ -47,4 +47,48 @@ public List getOperators(String ip, return operators; } + public List getPartners(String ip, + int port, + String username, + String password) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable partnerIds = highFlipClient.listPartners(0, 1000); + List partners = new ArrayList<>(); + for (String partnerId : partnerIds) { + Highflip.PartnerGetResponse partner = + highFlipClient.getPartner(partnerId); + partners.add(partner); + } + return partners; + } + + public List getDataList(String ip, + int port, + String username, + String password, + String targetPartnerId) { + // TODO: construct targetUri + String targetUriDemo = "grpc://username:password@host:port/path?query"; + // init highFlip client + HighFlipClient highFlipClient = + HighFlipClient.newHighFlipClient(targetUriDemo); + + Iterable dataIds = highFlipClient.listData(0, 1000); + List dataList = new ArrayList<>(); + for (String dataId : dataIds) { + Highflip.DataGetResponse data = highFlipClient.getData(dataId); + if (StringUtils.isBlank(targetPartnerId) || + (StringUtils.isNotBlank(data.getPartyId()) && + data.getPartyId().equals(targetPartnerId))) { + dataList.add(data); + } + } + return dataList; + } + } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index aaa80e2..e6de087 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -84,6 +84,7 @@ public void listConfig(Highflip.ConfigListRequest request, * @param request * @param responseObserver */ + @Override public void getConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -105,6 +106,7 @@ public void getConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void setConfig(Highflip.ConfigSetRequest request, StreamObserver responseObserver) { @@ -113,6 +115,7 @@ public void setConfig(Highflip.ConfigSetRequest request, returnVoid(responseObserver); } + @Override public void deleteConfig(Highflip.ConfigId request, StreamObserver responseObserver) { @@ -129,6 +132,7 @@ public void deleteConfig(Highflip.ConfigId request, * @param request * @param responseObserver */ + @Override public void getPlatform(Highflip.Void request, StreamObserver responseObserver) { @@ -147,6 +151,7 @@ public void getPlatform(Highflip.Void request, * @param request * @param responseObserver */ + @Override public void matchPlatform(Highflip.PlatformMatchRequest request, StreamObserver responseObserver) { @@ -189,6 +194,7 @@ public void createJob(Highflip.JobCreateRequest request, * @param request * @param responseObserver */ + @Override public void getJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); @@ -210,6 +216,7 @@ public void getJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void checkJob(Highflip.JobId request, StreamObserver responseObserver) { @@ -227,6 +234,7 @@ public void checkJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void deleteJob(Highflip.JobId request, StreamObserver responseObserver) { @@ -239,6 +247,7 @@ public void deleteJob(Highflip.JobId request, * @param request * @param responseObserver */ + @Override public void listJob(Highflip.JobListRequest request, StreamObserver responseObserver) { @@ -259,6 +268,7 @@ public void listJob(Highflip.JobListRequest request, * @param request * @param responseObserver */ + @Override public void controlJob(Highflip.JobControlRequest request, StreamObserver responseObserver) { @@ -270,6 +280,7 @@ public void controlJob(Highflip.JobControlRequest request, returnVoid(responseObserver); } + @Override public void getJobLog(Highflip.JobLogRequest request, StreamObserver responseObserver) { @@ -282,6 +293,7 @@ public void getJobLog(Highflip.JobLogRequest request, * @param request * @param responseObserver */ + @Override public void listTask(Highflip.TaskListRequest request, StreamObserver responseObserver) { @@ -300,6 +312,7 @@ public void listTask(Highflip.TaskListRequest request, * @param request * @param responseObserver */ + @Override public void getTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -320,6 +333,7 @@ public void getTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void checkTask(Highflip.TaskId request, StreamObserver responseObserver) { @@ -340,6 +354,7 @@ public void checkTask(Highflip.TaskId request, * @param request * @param responseObserver */ + @Override public void controlTask(Highflip.TaskControlRequest request, StreamObserver responseObserver) { @@ -355,6 +370,7 @@ public void controlTask(Highflip.TaskControlRequest request, * @param request * @param responseObserver */ + @Override public void getTaskLog(Highflip.TaskLogRequest request, StreamObserver responseObserver) { @@ -368,6 +384,7 @@ public void getTaskLog(Highflip.TaskLogRequest request, * @param request * @param responseObserver */ + @Override public void listData(Highflip.DataListRequest request, StreamObserver responseObserver) { @@ -383,6 +400,7 @@ public void listData(Highflip.DataListRequest request, * @param request * @param responseObserver */ + @Override public void getData(Highflip.DataId request, StreamObserver responseObserver) { @@ -413,6 +431,7 @@ public void getData(Highflip.DataId request, * @param responseObserver * @return */ + @Override public StreamObserver pushData( StreamObserver responseObserver) { @@ -496,6 +515,7 @@ public void onCompleted() { * @param request * @param responseObserver */ + @Override public void pullData(Highflip.DataPullRequest request, StreamObserver responseObserver) { @@ -537,6 +557,7 @@ public void pullData(Highflip.DataPullRequest request, * @param request * @param responseObserver */ + @Override public void deleteData(Highflip.DataId request, StreamObserver responseObserver) { @@ -552,6 +573,7 @@ public void deleteData(Highflip.DataId request, * @param request * @param responseObserver */ + @Override public void listOperator(Highflip.OperatorListRequest request, StreamObserver responseObserver) { Iterator response = Streams @@ -569,6 +591,7 @@ public void listOperator(Highflip.OperatorListRequest request, * @param request * @param responseObserver */ + @Override public void getOperator(Highflip.OperatorId request, StreamObserver responseObserver) { @@ -594,6 +617,7 @@ public void getOperator(Highflip.OperatorId request, * @param request * @param responseObserver */ + @Override public void createPartner(Highflip.PartnerCreateRequest request, StreamObserver responseObserver) { @@ -614,6 +638,7 @@ public void createPartner(Highflip.PartnerCreateRequest request, * @param request * @param responseObserver */ + @Override public void getPartner(Highflip.PartnerId request, StreamObserver responseObserver) { @@ -635,6 +660,7 @@ public void getPartner(Highflip.PartnerId request, * @param request * @param responseObserver */ + @Override public void listPartner(Highflip.PartnerListRequest request, StreamObserver responseObserver) { @@ -653,6 +679,7 @@ public void listPartner(Highflip.PartnerListRequest request, * @param request * @param responseObserver */ + @Override public void controlPartner(Highflip.PartnerControlRequest request, StreamObserver responseObserver) { From d8952cb9404027dfe71f2526997b702dd0c356f8 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 27 Apr 2023 11:27:24 +0800 Subject: [PATCH 05/17] enable job DAG save and retrieve --- .../highflip/editor/vo/SaveJobRequest.java | 13 +++++++++ .../web/controller/HighFlipController.java | 29 ++++++++++++++++++- .../web/services/HighFlipEditorService.java | 21 +++++++++++++- 3 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java new file mode 100644 index 0000000..e25ec31 --- /dev/null +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/vo/SaveJobRequest.java @@ -0,0 +1,13 @@ +package com.baidu.highflip.editor.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class SaveJobRequest { + + String json; +} diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java index 7a7b0c3..72169cc 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -2,6 +2,8 @@ import java.util.List; +import javax.servlet.http.HttpServletRequest; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -9,9 +11,11 @@ import org.springframework.web.bind.annotation.RestController; import com.baidu.highflip.editor.model.Result; +import com.baidu.highflip.editor.utils.StringUtils; import com.baidu.highflip.editor.vo.ListDataRequest; import com.baidu.highflip.editor.vo.ListOperatorsRequest; import com.baidu.highflip.editor.vo.ListPartnersRequest; +import com.baidu.highflip.editor.vo.SaveJobRequest; import com.baidu.highflip.editor.web.services.HighFlipEditorService; import com.baidu.highflip.editor.vo.LoginRequest; import com.baidu.highflip.editor.web.annotation.IgnoreRequestBody; @@ -19,13 +23,14 @@ import highflip.v1.Highflip; @RestController +@RequestMapping(value = "highflip") public class HighFlipController { @Autowired HighFlipEditorService highFlipEditorService; @IgnoreRequestBody - @RequestMapping(value = "/login", method = RequestMethod.POST) + @RequestMapping(value = "login", method = RequestMethod.POST) public Result login(@RequestBody LoginRequest request) { highFlipEditorService.login(request.getUsername(), request.getPassword(), @@ -73,4 +78,26 @@ public Result listData(ListDataRequest request) { username, password, request.getPartnerId()); return Result.success(dataGetResponseList); } + + @RequestMapping(value = "job", method = RequestMethod.POST) + public Result saveJob(HttpServletRequest request, + @RequestBody SaveJobRequest saveJobRequest) { + String token = request.getHeader("TOKEN"); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + highFlipEditorService.saveJob(token, saveJobRequest.getJson()); + return Result.success(); + } + + @RequestMapping(value = "job", method = RequestMethod.GET) + public Result getJob(HttpServletRequest request) { + String token = request.getHeader("TOKEN"); + if (StringUtils.isBlank(token)) { + return Result.failure(400, "The required header TOKEN has not found"); + } + String jobDag = highFlipEditorService.getJob(token); + return Result.success(jobDag); + } + } diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java index 5a9e4fd..9e274dc 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/services/HighFlipEditorService.java @@ -2,12 +2,14 @@ import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import com.baidu.highflip.client.HighFlipClient; +import com.baidu.highflip.editor.model.HighFlipEditorException; import com.baidu.highflip.editor.utils.StringUtils; import highflip.v1.Highflip; @@ -18,8 +20,10 @@ public class HighFlipEditorService { private static final Logger LOG = LoggerFactory.getLogger(HighFlipEditorService.class); - public HighFlipEditorService() { + private ConcurrentHashMap jobs; + public HighFlipEditorService() { + jobs = new ConcurrentHashMap<>(); } public void login(String username, String password, @@ -91,4 +95,19 @@ public List getDataList(String ip, return dataList; } + // save job + public void saveJob(String key, String jsonOfJob) { + if (jobs == null) { + jobs = new ConcurrentHashMap<>(); + } + jobs.put(key, jsonOfJob); + } + + public String getJob(String key){ + if(StringUtils.isBlank(key)){ + throw new HighFlipEditorException("Key is blank"); + } + return jobs.get(key); + } + } From fc3a669eb46a59d9383c2c08eeaf64226dcf7c29 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 27 Apr 2023 14:43:11 +0800 Subject: [PATCH 06/17] modify the header field --- .../highflip/editor/web/controller/HighFlipController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java index 72169cc..3300b95 100644 --- a/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java +++ b/highflip-editor/src/main/java/com/baidu/highflip/editor/web/controller/HighFlipController.java @@ -26,6 +26,8 @@ @RequestMapping(value = "highflip") public class HighFlipController { + private static String TOKEN = "Token"; + @Autowired HighFlipEditorService highFlipEditorService; @@ -82,7 +84,7 @@ public Result listData(ListDataRequest request) { @RequestMapping(value = "job", method = RequestMethod.POST) public Result saveJob(HttpServletRequest request, @RequestBody SaveJobRequest saveJobRequest) { - String token = request.getHeader("TOKEN"); + String token = request.getHeader(TOKEN); if (StringUtils.isBlank(token)) { return Result.failure(400, "The required header TOKEN has not found"); } @@ -92,7 +94,7 @@ public Result saveJob(HttpServletRequest request, @RequestMapping(value = "job", method = RequestMethod.GET) public Result getJob(HttpServletRequest request) { - String token = request.getHeader("TOKEN"); + String token = request.getHeader(TOKEN); if (StringUtils.isBlank(token)) { return Result.failure(400, "The required header TOKEN has not found"); } From 88f639360d127b6333d49331977cbc3c209e2224 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Fri, 12 May 2023 17:42:10 +0800 Subject: [PATCH 07/17] 1.test highflip dag and construct dag 2.add the convert of ListProto to java List and MapProto to java Map 3.modify the uuid strategy 4.register highflip runtime oject 5.complete HighFlipService 6.add highflip.adaptor.service.properties --- .../baidu/highflip/client/HighFlipClient.java | 7 + .../highflip/core/adaptor/ServiceAdaptor.java | 10 + .../core/common/AdaptorPropsList.java | 12 + .../core/common/InstanceNameList.java | 3 + .../highflip/core/engine/HighFlipRuntime.java | 2 + .../baidu/highflip/core/entity/dag/Graph.java | 4 +- .../baidu/highflip/core/entity/dag/Node.java | 8 +- .../baidu/highflip/core/entity/dag/Party.java | 1 + .../core/entity/dag/codec/TypeValue.java | 93 ++++- .../dag/common/NamedAttributeObject.java | 7 +- .../core/entity/dag/common/NodeInputRef.java | 6 +- .../core/entity/dag/common/NodeOutputRef.java | 6 +- .../highflip/core/entity/runtime/Data.java | 4 +- .../highflip/core/entity/runtime/Job.java | 4 +- .../core/entity/runtime/Operator.java | 2 +- .../highflip/core/entity/runtime/Partner.java | 2 +- .../core/entity/runtime/Platform.java | 2 +- .../highflip/core/entity/runtime/Task.java | 2 +- .../core/utils/CustomUuidGenerator.java | 24 ++ .../highflip/core/utils/SerializerUtils.java | 64 ++++ .../baidu/highflip/core/entity/TestDag.java | 360 +++++++++++++++++- .../src/main/proto/highflip-meta.proto | 4 + highflip-proto/src/main/proto/highflip.proto | 12 + .../server/adaptor/DefaultAdaptor.java | 9 + .../impl/ConfigurableServiceAdaptor.java | 42 ++ .../highflip/server/config/AdaptorConfig.java | 7 + .../server/engine/HighFlipEngine.java | 2 - .../engine/component/HighFlipContext.java | 6 + .../engine/component/HighFlipRuntime.java | 6 + .../server/rpc/v1/HighFlipRpcService.java | 21 +- .../adaptor/highflip.service.properties | 3 + .../src/main/resources/application.properties | 6 +- .../server/engine/HighFlipEngineTest.java | 5 + 33 files changed, 720 insertions(+), 26 deletions(-) create mode 100644 highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java create mode 100644 highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java create mode 100644 highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java create mode 100644 highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java create mode 100644 highflip-server/src/main/resources/adaptor/highflip.service.properties create mode 100644 highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 8a68fd9..8a4694e 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -584,4 +584,11 @@ public String createPartner(String name, String description){ return response.getPartnerId(); } + + public Highflip.GetServiceConfigResponse getServiceConfig() { + final Highflip.GetServiceConfigResponse serviceConfig = + getBlockingStub().getServiceConfig( + Highflip.Void.getDefaultInstance()); + return serviceConfig; + } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java new file mode 100644 index 0000000..104cfc1 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/adaptor/ServiceAdaptor.java @@ -0,0 +1,10 @@ +package com.baidu.highflip.core.adaptor; + +public interface ServiceAdaptor { + + String getUrl(); + + String getPartyId(); + + String getRole(); +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java index 70522ff..f156486 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/AdaptorPropsList.java @@ -25,4 +25,16 @@ public class AdaptorPropsList { public static final String PROPS_HIGHFLIP_ADAPTOR_PLATFORM_VERSION_DEFAULT = "0.0.0"; + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL = "highflip.adaptor.service.url"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT = "http://127.0.0.1:9380"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID = "highflip.adaptor.service.party.id"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT = "9999"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE = "highflip.adaptor.service.role"; + + public static final String PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT = "guest"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java index ef166ec..1a34c7f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/common/InstanceNameList.java @@ -25,4 +25,7 @@ public class InstanceNameList { public static final String HIGHFLIP_ADAPTOR_PARTNER = "highflip.adaptor.partner"; public static final String HIGHFLIP_ADAPTOR_USER = "highflip.adaptor.user"; + + public static final String HIGHFLIP_ADAPTOR_SERVICE = "highflip.adaptor.service"; + } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java index 26f069a..fc4ef77 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java @@ -26,4 +26,6 @@ public interface HighFlipRuntime { User getUser(String userId); Operator getOperator(String operatorId); + + Data registerData(Data data); } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java index 2224ee4..73eacfe 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Graph.java @@ -3,6 +3,8 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; import lombok.NoArgsConstructor; @@ -96,7 +98,7 @@ public Iterable listParties() { return getParties().keySet(); } - + @JsonIgnore protected void setNodeCategory() { calcMiddleNodes(); calcOutputNodes(); diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java index 2089128..78d73c0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Node.java @@ -6,16 +6,19 @@ import com.baidu.highflip.core.entity.dag.common.NodeInputRef; import com.baidu.highflip.core.entity.dag.common.NodeOutputRef; import com.baidu.highflip.core.utils.ProtoUtils; +import com.fasterxml.jackson.annotation.JsonIgnore; + import highflip.HighflipMeta; import lombok.Data; -import javax.persistence.Transient; import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.springframework.data.annotation.Transient; + @Data public class Node extends NamedAttributeObject implements Serializable { @@ -25,7 +28,7 @@ public class Node extends NamedAttributeObject implements Serializable { String description; - @Transient + @JsonIgnore Graph graph; Category category; @@ -108,6 +111,7 @@ public Node getInputNode(String name) { getInputs().get(name).getFromNode()); } + @JsonIgnore public List getInputNodes() { return getInputs() .values() diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java index c96e552..5d096f0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/Party.java @@ -3,6 +3,7 @@ import com.baidu.highflip.core.entity.dag.codec.AttributeMap; import com.baidu.highflip.core.entity.dag.common.NamedAttributeObject; import com.baidu.highflip.core.utils.ProtoUtils; + import highflip.HighflipMeta; import lombok.Data; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java index b5227f4..1a43401 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/codec/TypeValue.java @@ -1,5 +1,12 @@ package com.baidu.highflip.core.entity.dag.codec; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.protobuf.ByteString; + import highflip.HighflipMeta; public class TypeValue { @@ -17,11 +24,76 @@ public static Object fromProto(HighflipMeta.TypedValueProto value) { return value.getValue().getFloat(); } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.STRING.getNumber()) { return value.getValue().getString(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.BYTES.getNumber()) { + return value.getValue().getBytes(); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.LIST.getNumber()) { + return listProtoConvert2List(value.getValue().getList()); + } else if (typeValue == HighflipMeta.TypedValueProto.TypeProto.MAP.getNumber()) { + return mapProtoConvert2Map(value.getValue().getMap()); } else { throw new IllegalArgumentException(); } } + private static List listProtoConvert2List( + HighflipMeta.ListProto listProto) { + if (listProto == null) { + return new ArrayList<>(); + } + List result = new ArrayList<>(); + List listProtoList = listProto.getListList(); + for (HighflipMeta.ValueProto valueProto : listProtoList) { + result.add(convertValueProto(valueProto)); + } + return result; + } + + private static Map mapProtoConvert2Map(HighflipMeta.MapProto mapProto) { + if (mapProto == null) { + return new HashMap<>(); + } + Map map = mapProto.getMapMap(); + Map result = new HashMap<>(); + for (Map.Entry entry : + map.entrySet()) { + String key = entry.getKey(); + HighflipMeta.ValueProto value = entry.getValue(); + result.put(key, convertValueProto(value)); + } + return result; + } + + private static Object convertValueProto(HighflipMeta.ValueProto value) { + switch (value.getValueCase()) { + case BOOL: + return value.getBool(); + case INT: + return value.getInt(); + case LONG: + return value.getLong(); + case FLOAT: + return value.getFloat(); + case DOUBLE: + return value.getDouble(); + case STRING: + return value.getString(); + case BYTES: + return value.getBytes(); + case LIST: + List list = + value.getList().getListList(); + List resultList = new ArrayList<>(); + for(HighflipMeta.ValueProto listValue: list) { + resultList.add(convertValueProto(listValue)); + } + return resultList; + case MAP: + return mapProtoConvert2Map(value.getMap()); + default: + throw new RuntimeException("NOT_SUPPORTED_TYPE"); + } + } + public static HighflipMeta.TypedValueProto toProto(Object object) { HighflipMeta.TypedValueProto.Builder builder = HighflipMeta.TypedValueProto .newBuilder(); @@ -56,14 +128,31 @@ public static HighflipMeta.TypedValueProto toProto(Object object) { .newBuilder() .setDouble((Double) object) .build()); - } else { + } else if (object instanceof String) { builder.setType(HighflipMeta.TypedValueProto.TypeProto.STRING); builder.setValue(HighflipMeta.ValueProto .newBuilder() .setString(object.toString()) .build()); + } else if (object instanceof ByteString) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.BYTES); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setBytes((ByteString) object) + .build()); + } else if (object instanceof HighflipMeta.ListProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.LIST); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setList((HighflipMeta.ListProto) object) + .build()); + } else if (object instanceof HighflipMeta.MapProto) { + builder.setType(HighflipMeta.TypedValueProto.TypeProto.MAP); + builder.setValue(HighflipMeta.ValueProto + .newBuilder() + .setMap((HighflipMeta.MapProto) object) + .build()); } - return builder.build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java index e0f1377..0b9f14a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NamedAttributeObject.java @@ -2,15 +2,18 @@ import lombok.Data; -import javax.persistence.Transient; import java.util.List; import java.util.Map; import java.util.TreeMap; +import org.springframework.data.annotation.Transient; + +import com.fasterxml.jackson.annotation.JsonIgnore; + @Data public class NamedAttributeObject implements Comparable { - @Transient + @JsonIgnore List parents = List.of(); String name; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java index a6d9a63..d6b2eef 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeInputRef.java @@ -19,12 +19,15 @@ public class NodeInputRef { String fromOutput; + String value; + public static HighflipMeta.NodeInputProto toProto(com.baidu.highflip.core.entity.dag.common.NodeInputRef ref) { HighflipMeta.NodeInputProto.Builder builder = HighflipMeta.NodeInputProto .newBuilder() .setName(ref.getName()) .setFromNode(ref.getFromNode()) - .setFromOutput(ref.getFromOutput()); + .setFromOutput(ref.getFromOutput()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -37,6 +40,7 @@ public static com.baidu.highflip.core.entity.dag.common.NodeInputRef fromProto(H .setDescription(proto.getDescription()) .setFromNode(proto.getFromNode()) .setFromOutput(proto.getFromOutput()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java index e9fd012..270288a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/dag/common/NodeOutputRef.java @@ -15,10 +15,13 @@ public class NodeOutputRef { String description; + String value; + public static HighflipMeta.NodeOutputProto toProto(NodeOutputRef ref) { HighflipMeta.NodeOutputProto.Builder builder = HighflipMeta.NodeOutputProto .newBuilder() - .setName(ref.getName()); + .setName(ref.getName()) + .setValue(ref.getValue()); ProtoUtils.setOptional(builder, "Description", ProtoUtils.ofString(ref.getDescription())); return builder.build(); @@ -29,6 +32,7 @@ public static NodeOutputRef fromProto(HighflipMeta.NodeOutputProto proto) { .builder() .setName(proto.getName()) .setDescription(proto.getDescription()) + .setValue(proto.getValue()) .build(); } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java index 8086c0a..9ce80a0 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java @@ -32,8 +32,8 @@ public class Data { @Id - @Column(name = "data_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @Column(name = "data_id", length = 72) + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String dataId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java index 9d506fc..aaf9d3f 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Job.java @@ -37,7 +37,7 @@ public class Job { @Id @Column(name = "job_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String jobId; @@ -59,7 +59,7 @@ public class Job { DateTime finishTime; @Type(type = "json") - @Column(name = "graph") + @Column(name = "graph", length = 10240) Graph graph; @Column(name = "status") diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java index d8e9054..1af34ec 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Operator.java @@ -28,7 +28,7 @@ public class Operator { @Id @Column(name = "operator_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String operatorId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java index 41f25f2..6ab0f5a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Partner.java @@ -27,7 +27,7 @@ public class Partner { @Id @Column(name = "partner_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String partnerId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java index db5eaf6..8d7936a 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Platform.java @@ -33,7 +33,7 @@ public class Platform { @Id @Column(name = "platform_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String platformId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java index 07c3e35..5feae39 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java @@ -31,7 +31,7 @@ public class Task { @Id @Column(name = "task_id", length = 36) - @GenericGenerator(name = "id_gen", strategy = "uuid2") + @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String taskid; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java new file mode 100644 index 0000000..4e9ea66 --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/CustomUuidGenerator.java @@ -0,0 +1,24 @@ +package com.baidu.highflip.core.utils; + +import java.io.Serializable; + +import org.hibernate.HibernateException; +import org.hibernate.engine.spi.SharedSessionContractImplementor; +import org.hibernate.id.UUIDGenerator; + +public class CustomUuidGenerator extends UUIDGenerator { + + @Override + public Serializable generate(SharedSessionContractImplementor s, Object obj) + throws HibernateException { + Serializable id = s.getEntityPersister(null, obj).getClassMetadata() + .getIdentifier(obj, s); + + if (id != null && String.valueOf(id).length() > 0) { + return id; + } else { + return super.generate(s, obj); + } + } + +} diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java new file mode 100644 index 0000000..d252f4a --- /dev/null +++ b/highflip-core/src/main/java/com/baidu/highflip/core/utils/SerializerUtils.java @@ -0,0 +1,64 @@ +package com.baidu.highflip.core.utils; + +import java.util.HashMap; +import java.util.Map; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import org.springframework.lang.Nullable; + +public class SerializerUtils { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private SerializerUtils() { + } + + public static Map mapObjectDeserialize(String object, + Class tClass) + throws JsonProcessingException { + if (object == null || object.isBlank()) { + return null; + } + Map ret = new HashMap<>(); + Map map = objectMapper.readValue(object, + new TypeReference>() { + }); + for (String tempKey : map.keySet()) { + Object tempValue = map.get(tempKey); + if (tempValue != null) { + String decodeValue = objectMapper.writeValueAsString(tempValue); + if (tClass.isAssignableFrom(String.class)) { + //noinspection unchecked + ret.put(tempKey, (T) decodeValue); + } else { + T result = objectMapper.readValue(decodeValue, tClass); + ret.put(tempKey, result); + } + } + } + return ret; + } + + public static String toJsonString(Object object) + throws JsonProcessingException { + return objectMapper.writeValueAsString(object); + } + + public static T deserialize(String json, Class tClass) + throws JsonProcessingException { + return objectMapper.readValue(json, tClass); + } + + @Nullable + public static T deserializeType(String json, + TypeReference typeReference) + throws JsonProcessingException { + if (json == null || json.isBlank()) { + return null; + } + return objectMapper.readValue(json, typeReference); + } +} diff --git a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java index 7584d1a..b7f2135 100644 --- a/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java +++ b/highflip-core/src/test/java/com/baidu/highflip/core/entity/TestDag.java @@ -6,12 +6,18 @@ import com.baidu.highflip.core.entity.dag.PartyNode; import com.baidu.highflip.core.entity.dag.codec.TypeValue; import com.baidu.highflip.core.entity.dag.common.MappedNode; +import com.baidu.highflip.core.utils.SerializerUtils; +import com.fasterxml.jackson.core.JsonProcessingException; + import highflip.HighflipMeta; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; @Slf4j @@ -19,10 +25,7 @@ public class TestDag { @Test public void testGraph() { - HighflipMeta.GraphProto define = HighflipMeta.GraphProto - .newBuilder() - .setName("test_graph") - .build(); + HighflipMeta.GraphProto define = constructHighflipGraph(); var graph = Graph.fromProto(define); log.info("graph = {}", graph); @@ -229,4 +232,353 @@ public void testDag() { var proto = Graph.toProto(graph); log.info("proto = {}", proto); } + + private HighflipMeta.GraphProto constructHighflipGraph() { + // reader + List readerNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto outputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + readerNodeOutputProtos.add(outputProto); + + HighflipMeta.NodeProto reader = + HighflipMeta.NodeProto.newBuilder().setName("reader_0") + .setType("Reader") + .addAllOutputs(readerNodeOutputProtos) + .build(); + + // DataTransform + List dataTransformNodeOutputProtos = + new ArrayList<>(); + + HighflipMeta.NodeOutputProto dataTransformOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + HighflipMeta.NodeOutputProto dataTransformOutputProto2 = + HighflipMeta.NodeOutputProto.newBuilder().setName("model").setValue(getJsonStringOfModelOutput()).build(); + dataTransformNodeOutputProtos.add(dataTransformOutputProto); + dataTransformNodeOutputProtos.add(dataTransformOutputProto2); + + + + List dataTransformNodeInputProtos = + new ArrayList<>(); + String json = getDataTransformInputJsonString(); + HighflipMeta.NodeInputProto dataTransformInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(json) + .build(); + + + dataTransformNodeInputProtos.add(dataTransformInputProto); + + HighflipMeta.NodeProto dataTransform = + HighflipMeta.NodeProto.newBuilder().setName("data_transform_0") + .setType("DataTransform") + .addAllInputs( + dataTransformNodeInputProtos) + .addAllOutputs( + dataTransformNodeOutputProtos) + .build(); + + // intersect + List intersectNodeOutputProtos = + new ArrayList<>(); + HighflipMeta.NodeOutputProto intersectOutputProto = + HighflipMeta.NodeOutputProto.newBuilder().setName("data").setValue(getJsonStringOfDataOutput()).build(); + intersectNodeOutputProtos.add(intersectOutputProto); + + List intersectNodeInputProtos = + new ArrayList<>(); + + String dataTransformInputJsonString = getIntersectionInputJsonString(); + HighflipMeta.NodeInputProto intersectInputProto = + HighflipMeta.NodeInputProto.newBuilder() + .setName("data") + .setValue(dataTransformInputJsonString) + .build(); + + intersectNodeInputProtos.add(intersectInputProto); + + HighflipMeta.NodeProto intersect = + HighflipMeta.NodeProto.newBuilder().setName("intersect_0") + .setType("Intersection") + .addAllInputs(intersectNodeInputProtos) + .addAllOutputs(intersectNodeOutputProtos) + .build(); + + + // Map tableMap = new HashMap<>(); + // tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + // tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto mapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + // HighflipMeta.ValueProto valueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + // highflip.HighflipMeta.TypedValueProto typedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getGuestPartyReaderNode()) + .build(); + // + HighflipMeta.TypedValueProto withLabel= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + + HighflipMeta.TypedValueProto outputFormat= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("dense") + .build()).build(); + + HighflipMeta.PartyProto.PartyNode guestPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + // common conf + /* + + "intersect_0": { + "intersect_method": "rsa", + "sync_intersect_ids": false, + "only_output_key": true, + "rsa_params": { + "hash_method": "sha256", + "final_hash_method": "sha256", + "split_calculation": false, + "key_length": 2048 + } + } + */ + HighflipMeta.TypedValueProto intersectMethod= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.STRING_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setString("rsa") + .build()).build(); + HighflipMeta.TypedValueProto syncIntersectIds= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(false) + .build()).build(); + HighflipMeta.TypedValueProto onlyOutputKey= + HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.BOOLEAN_VALUE) + .setValue(HighflipMeta.ValueProto.newBuilder() + .setBool(true) + .build()).build(); + + // Map commonTableMap = + // new HashMap<>(); + + // tableMap.put("hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // tableMap.put("split_calculation", HighflipMeta.ValueProto.newBuilder().setBool(false).build()); + // tableMap.put("key_length", HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + + + + + // Map commonRsaParamConfMap = + // new HashMap<>(); + // commonRsaParamConfMap.put("hash_method", + // HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + // commonRsaParamConfMap.put("split_calculation", + // HighflipMeta.ValueProto.newBuilder().setBool( + // false).build()); + // commonRsaParamConfMap.put("key_length", + // HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + // HighflipMeta.MapProto commonValueMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + // HighflipMeta.ValueProto commonValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + // highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + + HighflipMeta.PartyProto.PartyNode commonPartyNode3 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(intersect.getName()) + .putAttributes("intersect_method",intersectMethod) + .putAttributes("sync_intersect_ids",syncIntersectIds) + .putAttributes("only_output_key",onlyOutputKey) + .putAttributes("rsa_params", + getCommonConfRsaParamTypedValueProto()) + .build(); + + List guestPartyNodes = + new ArrayList<>(); + guestPartyNodes.add(guestPartyNode1); + guestPartyNodes.add(guestPartyNode2); + guestPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto guest = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.GUEST).addAllNodes(guestPartyNodes).setName("9999").build(); + // guest 已经组装好 + + // 开始组装host + + // Map hostTableMap = + // new HashMap<>(); + // hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + // hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + // HighflipMeta.MapProto hostMapProto = + // HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + // HighflipMeta.ValueProto hostValueProto = + // HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + // highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + // highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + // HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + + HighflipMeta.PartyProto.PartyNode hostPartyNode1 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(reader.getName()) + .putAttributes("table", + getHostReaderTypedValueProto()) + .build(); + + // + HighflipMeta.PartyProto.PartyNode hostPartyNode2 = + HighflipMeta.PartyProto.PartyNode.newBuilder() + .setName(dataTransform.getName()) + .putAttributes("with_label",withLabel) + .putAttributes("output_format",outputFormat) + .build(); + List hostPartyNodes = + new ArrayList<>(); + hostPartyNodes.add(hostPartyNode1); + hostPartyNodes.add(hostPartyNode2); + hostPartyNodes.add(commonPartyNode3); + + HighflipMeta.PartyProto host = + HighflipMeta.PartyProto.newBuilder().setRole( + HighflipMeta.PartyRole.HOST).addAllNodes(hostPartyNodes).setName("10000").build(); + + List node = new ArrayList<>(); + node.add(reader); + node.add(dataTransform); + node.add(intersect); + + List partyProtoList = new ArrayList<>(); + partyProtoList.add(guest); + partyProtoList.add(host); + + HighflipMeta.GraphProto graphProto = + HighflipMeta.GraphProto.newBuilder().setName("jobId-202301") + .addAllNodes(node) + .addAllParties(partyProtoList).build(); + + return graphProto; + } + + private String getIntersectionInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("data_transform_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getDataTransformInputJsonString() { + Map> map = new HashMap<>(); + List data = new ArrayList<>(); + data.add("reader_0.data"); + map.put("data", data); + try { + return SerializerUtils.toJsonString(map); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfDataOutput() { + List data = new ArrayList<>(); + data.add("data"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private String getJsonStringOfModelOutput() { + List data = new ArrayList<>(); + data.add("model"); + try { + return SerializerUtils.toJsonString(data); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + private HighflipMeta.TypedValueProto getGuestPartyReaderNode() { + Map tableMap = new HashMap<>(); + tableMap.put("name", HighflipMeta.ValueProto.newBuilder().setString("csv1").build()); + tableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto mapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(tableMap).build(); + HighflipMeta.ValueProto valueProto = + HighflipMeta.ValueProto.newBuilder().setMap(mapProto).build(); + highflip.HighflipMeta.TypedValueProto typedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(valueProto).build(); + + return typedValueProto; + } + + private HighflipMeta.TypedValueProto getCommonConfRsaParamTypedValueProto(){ + Map commonRsaParamConfMap = + new HashMap<>(); + commonRsaParamConfMap.put("hash_method", + HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("final_hash_method", HighflipMeta.ValueProto.newBuilder().setString("sha256").build()); + commonRsaParamConfMap.put("split_calculation", + HighflipMeta.ValueProto.newBuilder().setBool( + false).build()); + commonRsaParamConfMap.put("key_length", + HighflipMeta.ValueProto.newBuilder().setInt(2048).build()); + HighflipMeta.MapProto commonValueMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(commonRsaParamConfMap).build(); + HighflipMeta.ValueProto commonValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(commonValueMapProto).build(); + highflip.HighflipMeta.TypedValueProto commonTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(commonValueProto).build(); + return commonTypedValueProto; + } + + private HighflipMeta.TypedValueProto getHostReaderTypedValueProto() { + Map hostTableMap = + new HashMap<>(); + hostTableMap.put("name",HighflipMeta.ValueProto.newBuilder().setString("csv2").build()); + hostTableMap.put("namespace", HighflipMeta.ValueProto.newBuilder().setString("HIGH-FLIP").build()); + HighflipMeta.MapProto hostMapProto = + HighflipMeta.MapProto.newBuilder().putAllMap(hostTableMap).build(); + HighflipMeta.ValueProto hostValueProto = + HighflipMeta.ValueProto.newBuilder().setMap(hostMapProto).build(); + highflip.HighflipMeta.TypedValueProto hostTypedValueProto = + highflip.HighflipMeta.TypedValueProto.newBuilder().setTypeValue( + HighflipMeta.TypedValueProto.TypeProto.MAP_VALUE).setValue(hostValueProto).build(); + return hostTypedValueProto; + } + } diff --git a/highflip-proto/src/main/proto/highflip-meta.proto b/highflip-proto/src/main/proto/highflip-meta.proto index a093630..a11552c 100644 --- a/highflip-proto/src/main/proto/highflip-meta.proto +++ b/highflip-proto/src/main/proto/highflip-meta.proto @@ -65,6 +65,8 @@ message NodeInputProto{ string from_node = 4; string from_output = 5; + + string value = 6; } message NodeOutputProto{ @@ -74,6 +76,8 @@ message NodeOutputProto{ string name = 2; optional string description = 3; + + string value = 4; } message NodeProto{ diff --git a/highflip-proto/src/main/proto/highflip.proto b/highflip-proto/src/main/proto/highflip.proto index 880a5b6..bdaabba 100644 --- a/highflip-proto/src/main/proto/highflip.proto +++ b/highflip-proto/src/main/proto/highflip.proto @@ -559,6 +559,15 @@ message ConfigListResponse{ string key = 1; } +message GetServiceConfigResponse{ + + string url = 1; + + string party_id = 2; + + string role = 3; +} + //////////////////////////////////////////////////////////////////////////////// // SERVICE @@ -608,4 +617,7 @@ service HighFlip{ rpc getPartner(PartnerId) returns(PartnerGetResponse); rpc listPartner(PartnerListRequest) returns(stream PartnerListResponse); rpc controlPartner(PartnerControlRequest) returns(Void); + + // adaptor service config + rpc getServiceConfig(Void) returns(GetServiceConfigResponse); } \ No newline at end of file diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java index a9fdf38..aa6dde4 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java @@ -6,6 +6,7 @@ import com.baidu.highflip.server.adaptor.impl.ConfigurableOperatorAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePartnerAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePlatformAdaptor; +import com.baidu.highflip.server.adaptor.impl.ConfigurableServiceAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbJobAdaptor; import com.baidu.highflip.server.adaptor.impl.DumbTaskAdaptor; import com.baidu.highflip.server.adaptor.impl.FixedSingleDataAdaptor; @@ -22,6 +23,11 @@ public class DefaultAdaptor implements HighFlipAdaptor { public static final String PLATFORM_PROPERTIES = "/adaptor/highflip.platform.properties"; + /** + * 用于配置本侧服务的配置信息 + */ + public static final String SERVICE_PROPERTIES = "/adaptor/highflip.service.properties"; + @Override public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_JOB, new DumbJobAdaptor()); @@ -37,6 +43,9 @@ public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_PLATFORM, new ConfigurablePlatformAdaptor(HighFlipUtils.getProperty(PLATFORM_PROPERTIES))); + + register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, + new ConfigurableServiceAdaptor(HighFlipUtils.getProperty(SERVICE_PROPERTIES))); } @Override diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java new file mode 100644 index 0000000..800bbc8 --- /dev/null +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/impl/ConfigurableServiceAdaptor.java @@ -0,0 +1,42 @@ +package com.baidu.highflip.server.adaptor.impl; + +import java.util.Properties; + +import com.baidu.highflip.core.adaptor.ServiceAdaptor; +import com.baidu.highflip.core.common.AdaptorPropsList; + +public class ConfigurableServiceAdaptor implements ServiceAdaptor { + + String url; + + String partyId; + + String role; + + public ConfigurableServiceAdaptor(Properties props) { + + this.url = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_URL_DEFAULT); + + this.partyId = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_PARTY_ID_DEFAULT); + + this.role = props.getProperty(AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE, + AdaptorPropsList.PROPS_HIGHFLIP_ADAPTOR_SERVICE_ROLE_DEFAULT); + } + + @Override + public String getUrl() { + return this.url; + } + + @Override + public String getPartyId() { + return this.partyId; + } + + @Override + public String getRole() { + return this.role; + } +} diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java index 0c99b7e..af0ea2e 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java @@ -1,8 +1,11 @@ package com.baidu.highflip.server.config; +import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.DefaultAdaptor; import com.baidu.highflip.server.adaptor.loader.AdaptorLoader; +import com.baidu.highflip.server.engine.component.HighFlipRuntime; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -23,6 +26,9 @@ public class AdaptorConfig { @Autowired InstanceRegister register; + @Autowired + HighFlipRuntime highFlipRuntime; + AdaptorLoader loader = null; @PostConstruct @@ -73,5 +79,6 @@ void initialAdaptor() { void initialDefaultAdaptor() { DefaultAdaptor adaptor = new DefaultAdaptor(); adaptor.setup(register); + register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java index 525db92..2085b0a 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java @@ -290,7 +290,6 @@ public Job createJob(String name, String description, Graph graph) { job = getContext().getJobAdaptor() .createJob(job); - job = getContext().getJobRepository() .save(job); @@ -317,7 +316,6 @@ public Job createJob(String name, String description, Graph graph) { return job; } - // @Scheduled protected void updateJob() { diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java index d204511..3d2a07b 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipContext.java @@ -5,6 +5,7 @@ import com.baidu.highflip.core.adaptor.OperatorAdaptor; import com.baidu.highflip.core.adaptor.PartnerAdaptor; import com.baidu.highflip.core.adaptor.PlatformAdaptor; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.adaptor.TaskAdaptor; import com.baidu.highflip.core.adaptor.UserAdaptor; import com.baidu.highflip.core.common.InstanceNameList; @@ -125,4 +126,9 @@ public UserAdaptor getUserAdaptor(){ public UserRepository getUserRepository() { return userReps; } + + // SERVICE + public ServiceAdaptor getServiceAdaptor() { + return getInstance(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java index 55c0412..934c929 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java @@ -69,4 +69,10 @@ public User getUser(String userId) { public Operator getOperator(String operatorId) { return null; } + + @Override + public Data registerData(Data data) { + Data dataSaved = getContext().getDataRepository().save(data); + return dataSaved; + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index e6de087..e16cd85 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -1,5 +1,6 @@ package com.baidu.highflip.server.rpc.v1; +import com.baidu.highflip.core.adaptor.ServiceAdaptor; import com.baidu.highflip.core.entity.dag.Graph; import com.baidu.highflip.core.entity.runtime.Config; import com.baidu.highflip.core.entity.runtime.Data; @@ -221,12 +222,17 @@ public void checkJob(Highflip.JobId request, StreamObserver responseObserver) { Job job = getEngine().getJob(request.getJobId()); + com.baidu.highflip.core.entity.runtime.basic.Status jobStatus = + getEngine().getContext() + .getJobAdaptor() + .getJobStatus(job); Highflip.JobCheckResponse response = Highflip.JobCheckResponse .newBuilder() .setJobId(job.getJobId()) + .setStatus(Highflip.JobCheckResponse.JobStatus.valueOf( + jobStatus.toString().toUpperCase())) .build(); - returnOne(responseObserver, response); } @@ -685,4 +691,17 @@ public void controlPartner(Highflip.PartnerControlRequest request, } + @Override + public void getServiceConfig(Highflip.Void request, + StreamObserver responseObserver) { + final ServiceAdaptor serviceAdaptor = + getEngine().getContext().getServiceAdaptor(); + Highflip.GetServiceConfigResponse response = + Highflip.GetServiceConfigResponse.newBuilder() + .setUrl(serviceAdaptor.getUrl()) + .setPartyId(serviceAdaptor.getPartyId()) + .setRole(serviceAdaptor.getRole()) + .build(); + returnOne(responseObserver, response); + } } diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties new file mode 100644 index 0000000..b36b940 --- /dev/null +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -0,0 +1,3 @@ +highflip.adaptor.service.url=http://10.27.130.41:8380 +highflip.adaptor.service.party.id=9999 +highflip.adaptor.service.role=guest \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index e7c6ac5..975ae95 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path= +highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar ######################################################### # spring grpc ######################################################### @@ -30,7 +30,9 @@ spring.jpa.properties.jadira.usertype.autoRegisterUserTypes=true # h2 spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect spring.datasource.driver-class-name=org.h2.Driver -spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip.db +spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip +spring.datasource.username=highflip +spring.datasource.password=HighFlip@123 ######################################################### # logging ######################################################### diff --git a/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java new file mode 100644 index 0000000..d4651d8 --- /dev/null +++ b/highflip-server/src/test/java/com/baidu/highflip/server/engine/HighFlipEngineTest.java @@ -0,0 +1,5 @@ +package com.baidu.highflip.server.engine; + +public class HighFlipEngineTest { + +} From b32218a1f266143c4bbee07d69faa84f440ea020 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Mon, 31 Jul 2023 14:24:54 +0800 Subject: [PATCH 08/17] config github actions and deploy to maven central repository --- .github/workflows/maven.yml | 24 +- .../adaptor/highflip.service.properties | 6 +- .../src/main/resources/application.properties | 2 +- pom.xml | 208 ++++++++++++++++-- 4 files changed, 217 insertions(+), 23 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 1d25004..20f2f7f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,8 +27,30 @@ jobs: java-version: '11' distribution: 'temurin' cache: maven + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build with Maven - run: mvn -B package --file pom.xml + run: mvn -B -Prelease -DskipTests package --file pom.xml + - name: Create release + uses: ncipollo/release-action@v1 + with: + allowUpdates: true + artifacts: "${{ github.workspace }}/target/*.jar" + token: ${{ secrets.GITHUB_TOKEN }} + - name: Publish to the Maven Central Repository + run: | + mvn clean \ + --no-transfer-progress \ + --batch-mode \ + -Prelease \ + -DskipTests \ + deploy + env: + MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive # - name: Update dependency graph diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties index b36b940..07cc227 100644 --- a/highflip-server/src/main/resources/adaptor/highflip.service.properties +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -1,3 +1,3 @@ -highflip.adaptor.service.url=http://10.27.130.41:8380 -highflip.adaptor.service.party.id=9999 -highflip.adaptor.service.role=guest \ No newline at end of file +highflip.adaptor.service.url= +highflip.adaptor.service.party.id= +highflip.adaptor.service.role= \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index 975ae95..7f48543 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar +highflip.server.adaptor.path= ######################################################### # spring grpc ######################################################### diff --git a/pom.xml b/pom.xml index eb41ecc..468baf8 100644 --- a/pom.xml +++ b/pom.xml @@ -7,8 +7,8 @@ pom 1.0.0-SNAPSHOT highflip - - http://maven.apache.org + High layer Federated Learning Intercommunication Protocol + https://github.com/baidu/highflip @@ -29,6 +29,11 @@ 11 11 1.49.0 + + + 3.2.0 + 1.6 + 1.6.7 @@ -41,6 +46,36 @@ highflip-editor + + + dev + chenzhiyu + work123@Baidu + https://oss.sonatype.org/ + The HighFlip Project + https://oss.sonatype.org/ + + + + + + ossrh-snapshot + https://oss.sonatype.org/content/repositories/snapshots + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + https://github.com/baidu/highflip + scm:git:git://github.com/baidu/highflip.git + scm:git:ssh://git@github.com/baidu/highflip.git + HEAD + + @@ -120,6 +155,55 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadoc + + jar + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-source + + jar + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.2 + + + + prepare-agent + + + + + report + test + + report + + + + + + org.apache.maven.plugins + maven-jar-plugin + ${maven.jar.plugin.version} + @@ -130,23 +214,111 @@ org.jacoco jacoco-maven-plugin - 0.8.2 - - - - prepare-agent - - - - - report - test - - report - - - + + + org.apache.maven.plugins + maven-jar-plugin + + + + local + + true + + + + + release + + + + maven-source-plugin + + + create-source-jar + + jar + test-jar + + + + + + maven-jar-plugin + + + + test-jar + + + + true + + + + + + maven-javadoc-plugin + + + create-javadoc-jar + + javadoc + jar + + package + + + + ${java.version} + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven.gpg.plugin.version} + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${nexus.staging.maven.plugin.version} + true + + sonatype-nexus-repository + https://oss.sonatype.org + true + + + + + + + sonatype-nexus-repository + https://oss.sonatype.org/content/repositories/snapshots + + + sonatype-nexus-repository + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + \ No newline at end of file From 0f56b6663d1659dd927bd4ab4ca5f9eca23f8946 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 3 Aug 2023 11:21:57 +0800 Subject: [PATCH 09/17] modify code after review --- .github/workflows/maven.yml | 23 ++++++++---------- .../console/commands/TaskCommand.java | 5 ++-- .../baidu/highflip/client/HighFlipClient.java | 13 ++++------ .../highflip/core/engine/HighFlipRuntime.java | 5 ++++ .../highflip/core/entity/runtime/Data.java | 2 +- .../highflip/core/entity/runtime/Task.java | 9 +++++++ highflip-editor/pom.xml | 18 +++++++------- highflip-proto/src/main/proto/highflip.proto | 5 ++-- .../server/adaptor/DefaultAdaptor.java | 22 +++++++++++++++-- .../highflip/server/config/AdaptorConfig.java | 4 +--- .../server/engine/HighFlipEngine.java | 21 ++++++++++------ .../engine/component/HighFlipRuntime.java | 18 +++++++++++++- .../server/rpc/v1/HighFlipRpcService.java | 24 +++++++------------ .../adaptor/highflip.service.properties | 6 ++--- .../src/main/resources/application.properties | 4 ++-- pom.xml | 17 ++++++++----- 16 files changed, 119 insertions(+), 77 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 20f2f7f..f4f0969 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -13,6 +13,11 @@ on: branches: [ "master" ] pull_request: branches: [ "master" ] + workflow_dispatch: + inputs: + branch: + description: 'The branch to build' + required: true jobs: build: @@ -29,16 +34,8 @@ jobs: cache: maven server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }} + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: Build with Maven - run: mvn -B -Prelease -DskipTests package --file pom.xml - - name: Create release - uses: ncipollo/release-action@v1 - with: - allowUpdates: true - artifacts: "${{ github.workspace }}/target/*.jar" - token: ${{ secrets.GITHUB_TOKEN }} - name: Publish to the Maven Central Repository run: | mvn clean \ @@ -46,11 +43,11 @@ jobs: --batch-mode \ -Prelease \ -DskipTests \ - deploy + deploy --file pom.xml env: - MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} + MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive # - name: Update dependency graph diff --git a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java index 724295b..5c969f0 100644 --- a/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java +++ b/highflip-clients/highflip-console/src/main/java/com/baidu/highflip/console/commands/TaskCommand.java @@ -17,12 +17,13 @@ public class TaskCommand { @Autowired HighFlipClient client; - @ShellMethod(key = "task list", value = "List all task ids") + @ShellMethod(key = "task list", value = "List job all task ids") public Iterable list( + @ShellOption String jobId, @ShellOption(defaultValue = "0") Integer offset, @ShellOption(defaultValue = "0") Integer limit) { - return client.listTasks(offset, limit); + return client.listTasks(jobId, offset, limit); } @ShellMethod(key = "task get", value = "Get a task information") diff --git a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java index 8a4694e..3cd80e0 100644 --- a/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java +++ b/highflip-clients/highflip-sdk/src/main/java/com/baidu/highflip/client/HighFlipClient.java @@ -289,13 +289,17 @@ public Iterable getJobLog(String jobId){ /** * + * @param jobId job id in highflip * @param offset * @param limit * @return */ - public Iterable listTasks(int offset, int limit){ + public Iterable listTasks(String jobId, int offset, int limit) { Highflip.TaskListRequest request = Highflip.TaskListRequest .newBuilder() + .setJobId(jobId) + .setOffset(offset) + .setLimit(limit) .build(); Iterator response = getBlockingStub() @@ -584,11 +588,4 @@ public String createPartner(String name, String description){ return response.getPartnerId(); } - - public Highflip.GetServiceConfigResponse getServiceConfig() { - final Highflip.GetServiceConfigResponse serviceConfig = - getBlockingStub().getServiceConfig( - Highflip.Void.getDefaultInstance()); - return serviceConfig; - } } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java index fc4ef77..21a56a6 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/engine/HighFlipRuntime.java @@ -6,6 +6,7 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; public interface HighFlipRuntime { @@ -28,4 +29,8 @@ public interface HighFlipRuntime { Operator getOperator(String operatorId); Data registerData(Data data); + + Iterable listTask(String jobId); + + void updateTask(Task task); } diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java index 9ce80a0..bc4239b 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Data.java @@ -32,7 +32,7 @@ public class Data { @Id - @Column(name = "data_id", length = 72) + @Column(name = "data_id", length = 36) @GenericGenerator(name = "id_gen", strategy = "com.baidu.highflip.core.utils.CustomUuidGenerator") @GeneratedValue(generator = "id_gen") String dataId; diff --git a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java index 5feae39..f4ed6ad 100644 --- a/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java +++ b/highflip-core/src/main/java/com/baidu/highflip/core/entity/runtime/Task.java @@ -35,6 +35,10 @@ public class Task { @GeneratedValue(generator = "id_gen") String taskid; + /** + * job id is not the job id in highflip but the job id in corresponding + * federated learning service + */ @Column(name = "job_id") String jobid; @@ -81,4 +85,9 @@ public class Task { @Type(type = "json") @Column(name = "binding") Map binding; + + @Type(type = "json") + @Column(name = "output_data") + List outputData; + } diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index 7f2e216..9919f91 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -11,16 +11,14 @@ highflip-editor - - 11 - 11 - 2.7.10 - 1.18.22 - 6.11 - 1.43 - 31.1-jre - 6.0.2 - + + + + + + + + diff --git a/highflip-proto/src/main/proto/highflip.proto b/highflip-proto/src/main/proto/highflip.proto index bdaabba..119c543 100644 --- a/highflip-proto/src/main/proto/highflip.proto +++ b/highflip-proto/src/main/proto/highflip.proto @@ -230,6 +230,8 @@ message TaskGetResponse{ repeated string input_tasks = 9; repeated string output_tasks = 10; + + repeated string output_data_ids = 11; } @@ -617,7 +619,4 @@ service HighFlip{ rpc getPartner(PartnerId) returns(PartnerGetResponse); rpc listPartner(PartnerListRequest) returns(stream PartnerListResponse); rpc controlPartner(PartnerControlRequest) returns(Void); - - // adaptor service config - rpc getServiceConfig(Void) returns(GetServiceConfigResponse); } \ No newline at end of file diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java index aa6dde4..e183cd5 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/adaptor/DefaultAdaptor.java @@ -2,6 +2,7 @@ import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.HighFlipAdaptor; +import com.baidu.highflip.core.engine.HighFlipRuntime; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.impl.ConfigurableOperatorAdaptor; import com.baidu.highflip.server.adaptor.impl.ConfigurablePartnerAdaptor; @@ -28,6 +29,12 @@ public class DefaultAdaptor implements HighFlipAdaptor { */ public static final String SERVICE_PROPERTIES = "/adaptor/highflip.service.properties"; + private HighFlipRuntime highFlipRuntime; + + public DefaultAdaptor(HighFlipRuntime highFlipRuntime) { + this.highFlipRuntime = highFlipRuntime; + } + @Override public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_JOB, new DumbJobAdaptor()); @@ -44,8 +51,19 @@ public void setup(InstanceRegister register) { register.register(InstanceNameList.HIGHFLIP_ADAPTOR_PLATFORM, new ConfigurablePlatformAdaptor(HighFlipUtils.getProperty(PLATFORM_PROPERTIES))); - register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, - new ConfigurableServiceAdaptor(HighFlipUtils.getProperty(SERVICE_PROPERTIES))); + final ConfigurableServiceAdaptor configurableServiceAdaptor = + new ConfigurableServiceAdaptor( + HighFlipUtils.getProperty(SERVICE_PROPERTIES)); + register.register(InstanceNameList.HIGHFLIP_ADAPTOR_SERVICE, configurableServiceAdaptor); + + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.url", + configurableServiceAdaptor.getUrl()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.party.id", + configurableServiceAdaptor.getPartyId()); + highFlipRuntime.getConfiguration().setString("highflip.adaptor.service.role", + configurableServiceAdaptor.getRole()); + + register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } @Override diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java index af0ea2e..2a7827d 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/config/AdaptorConfig.java @@ -1,6 +1,5 @@ package com.baidu.highflip.server.config; -import com.baidu.highflip.core.common.InstanceNameList; import com.baidu.highflip.core.engine.InstanceRegister; import com.baidu.highflip.server.adaptor.DefaultAdaptor; import com.baidu.highflip.server.adaptor.loader.AdaptorLoader; @@ -77,8 +76,7 @@ void initialAdaptor() { } void initialDefaultAdaptor() { - DefaultAdaptor adaptor = new DefaultAdaptor(); + DefaultAdaptor adaptor = new DefaultAdaptor(highFlipRuntime); adaptor.setup(register); - register.register(InstanceNameList.HIGHFLIP_RUNTIME, highFlipRuntime); } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java index 2085b0a..05443c3 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/HighFlipEngine.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -64,7 +65,7 @@ public class HighFlipEngine { @Autowired PlatformTransactionManager transactionManager; - ConcurrentMap activeJobs; + ConcurrentMap activeJobs = new ConcurrentHashMap<>(); @Autowired AsyncTaskExecutor executor; @@ -313,11 +314,12 @@ public Job createJob(String name, String description, Graph graph) { getContext().getTaskRepository() .saveAll(news); } + activeJobs.putIfAbsent(job.getJobId(), job); return job; } // @Scheduled - protected void updateJob() { + public void updateJob() { JobAdaptor adaptor = getContext().getJobAdaptor(); @@ -326,6 +328,10 @@ protected void updateJob() { if (status != job.getStatus()) { job.setStatus(status); getContext().getJobRepository().save(job); + getContext().getJobAdaptor().updateJob(job); + } + if (status == Status.SUCCEEDED || status == Status.FAILED) { + activeJobs.remove(job.getJobId()); } }); } @@ -411,14 +417,15 @@ public void initializeTasks() { } // @Scheduled - private void updateTask() { - + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); } public Iterable listTask(String jobid) { - - return getContext().getTaskRepository() - .findAllByJobid(jobid); + final List tasks = getContext().getTaskRepository() + .findAllByJobid(jobid); + log.info("tasks size: {}", tasks.size()); + return tasks; } @Cacheable(value = "tasks") diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java index 934c929..791cce8 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/engine/component/HighFlipRuntime.java @@ -7,6 +7,8 @@ import com.baidu.highflip.core.entity.runtime.Partner; import com.baidu.highflip.core.entity.runtime.Task; import com.baidu.highflip.core.entity.runtime.User; +import com.baidu.highflip.core.entity.runtime.basic.Status; + import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -18,13 +20,16 @@ public class HighFlipRuntime implements com.baidu.highflip.core.engine.HighFlipR @Autowired HighFlipContext context; + @Autowired + HighFlipConfiguration configuration; + HighFlipContext getContext() { return context; } @Override public Configuration getConfiguration() { - return null; + return configuration; } @Override @@ -75,4 +80,15 @@ public Data registerData(Data data) { Data dataSaved = getContext().getDataRepository().save(data); return dataSaved; } + + @Override + public Iterable listTask(String jobId) { + return getContext().getTaskRepository() + .findAllByJobid(jobId); + } + + @Override + public void updateTask(Task task) { + getContext().getTaskRepository().save(task); + } } diff --git a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java index e16cd85..0640b8f 100644 --- a/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java +++ b/highflip-server/src/main/java/com/baidu/highflip/server/rpc/v1/HighFlipRpcService.java @@ -226,6 +226,8 @@ public void checkJob(Highflip.JobId request, getEngine().getContext() .getJobAdaptor() .getJobStatus(job); + // check and update job + getEngine().updateJob(); Highflip.JobCheckResponse response = Highflip.JobCheckResponse .newBuilder() @@ -302,9 +304,11 @@ public void getJobLog(Highflip.JobLogRequest request, @Override public void listTask(Highflip.TaskListRequest request, StreamObserver responseObserver) { - + final Job job = getEngine().getJob(request.getJobId()); + log.info("job id: {}, job binging id: {}", job.getJobId(), + job.getBingingId()); Iterator response = Streams - .stream(getEngine().listTask(request.getJobId())) + .stream(getEngine().listTask(job.getBingingId())) .map(t -> Highflip.TaskListResponse .newBuilder() .setTaskId(t.getTaskid()) @@ -328,8 +332,10 @@ public void getTask(Highflip.TaskId request, .newBuilder() .setTaskId(task.getTaskid()) .setJobId(task.getJobid()) + .setNodeName(task.getNodeName()) .setCreateTime(task.getCreateTime().toString()) .setUpdateTime(task.getUpdateTime().toString()) + .addAllOutputDataIds(task.getOutputData()) .build(); returnOne(responseObserver, response); @@ -690,18 +696,4 @@ public void controlPartner(Highflip.PartnerControlRequest request, StreamObserver responseObserver) { } - - @Override - public void getServiceConfig(Highflip.Void request, - StreamObserver responseObserver) { - final ServiceAdaptor serviceAdaptor = - getEngine().getContext().getServiceAdaptor(); - Highflip.GetServiceConfigResponse response = - Highflip.GetServiceConfigResponse.newBuilder() - .setUrl(serviceAdaptor.getUrl()) - .setPartyId(serviceAdaptor.getPartyId()) - .setRole(serviceAdaptor.getRole()) - .build(); - returnOne(responseObserver, response); - } } diff --git a/highflip-server/src/main/resources/adaptor/highflip.service.properties b/highflip-server/src/main/resources/adaptor/highflip.service.properties index 07cc227..b36b940 100644 --- a/highflip-server/src/main/resources/adaptor/highflip.service.properties +++ b/highflip-server/src/main/resources/adaptor/highflip.service.properties @@ -1,3 +1,3 @@ -highflip.adaptor.service.url= -highflip.adaptor.service.party.id= -highflip.adaptor.service.role= \ No newline at end of file +highflip.adaptor.service.url=http://10.27.130.41:8380 +highflip.adaptor.service.party.id=9999 +highflip.adaptor.service.role=guest \ No newline at end of file diff --git a/highflip-server/src/main/resources/application.properties b/highflip-server/src/main/resources/application.properties index 7f48543..48ed02f 100644 --- a/highflip-server/src/main/resources/application.properties +++ b/highflip-server/src/main/resources/application.properties @@ -1,7 +1,7 @@ ######################################################### # highflip adaptor ######################################################### -highflip.server.adaptor.path= +highflip.server.adaptor.path=file:/Users/songtingyu/tmp/fate_adaptor/highflip-adaptor-fate-1.0.0-SNAPSHOT-jar-with-dependencies.jar ######################################################### # spring grpc ######################################################### @@ -32,7 +32,7 @@ spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.H2Dialect spring.datasource.driver-class-name=org.h2.Driver spring.datasource.url=jdbc:h2:file:~/.highflip/db/highflip spring.datasource.username=highflip -spring.datasource.password=HighFlip@123 +spring.datasource.password=changeMe ######################################################### # logging ######################################################### diff --git a/pom.xml b/pom.xml index 468baf8..e78d907 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,12 @@ 11 11 1.49.0 + 2.7.3 + 1.18.22 + 6.11 + 1.43 + 31.1-jre + 6.0.2 3.2.0 @@ -48,12 +54,11 @@ - dev - chenzhiyu - work123@Baidu - https://oss.sonatype.org/ - The HighFlip Project - https://oss.sonatype.org/ + Baidu dianshi + dianshi-help@baidu.com + https://dianshi.baidu.com/ + dianshi + https://dianshi.baidu.com/ From 05d237007ffcf798576684c3b74e8fdcfbce79b3 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 3 Aug 2023 14:19:24 +0800 Subject: [PATCH 10/17] modify maven.yaml --- .github/workflows/maven.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f4f0969..652b01d 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -37,17 +37,17 @@ jobs: gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Publish to the Maven Central Repository - run: | - mvn clean \ - --no-transfer-progress \ - --batch-mode \ - -Prelease \ - -DskipTests \ - deploy --file pom.xml - env: - MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} + run: | + mvn clean \ + --no-transfer-progress \ + --batch-mode \ + -Prelease \ + -DskipTests \ + deploy --file pom.xml + env: + MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive # - name: Update dependency graph From db84ef79e2bdd14426b97759201a2735d17bae82 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 3 Aug 2023 15:51:32 +0800 Subject: [PATCH 11/17] modify maven.yml --- .github/workflows/maven.yml | 63 +++++++++++++++++++++++-------------- pom.xml | 36 ++++++++------------- 2 files changed, 53 insertions(+), 46 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 652b01d..9dd5638 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,29 +25,46 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - java-version: '11' - distribution: 'temurin' - cache: maven - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - - name: Publish to the Maven Central Repository - run: | - mvn clean \ - --no-transfer-progress \ - --batch-mode \ - -Prelease \ - -DskipTests \ - deploy --file pom.xml - env: - MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} - MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} - MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} + - name: Checkout project + uses: actions/checkout@v3 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' +# cache: maven +# server-username: MAVEN_USERNAME +# server-password: MAVEN_PASSWORD +# gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} +# gpg-passphrase: MAVEN_GPG_PASSPHRASE + + - name: Build with Maven + run: mvn -B package -DskipTests -Prelease --file pom.xml + + - name: Set up Apache Maven Central + uses: actions/setup-java@v3 + with: # running setup-java again overwrites the settings.xml + java-version: '11' + distribution: 'temurin' + server-id: ossrh # Value of the distributionManagement/repository/id field of the pom.xml + server-username: MAVEN_USERNAME # env variable for username in deploy + server-password: MAVEN_PASSWORD # env variable for token in deploy + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import + gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + + - name: Publish to the Maven Central Repository + run: | + mvn clean \ + --no-transfer-progress \ + --batch-mode \ + -Prelease \ + -DskipTests \ + deploy --file pom.xml + env: + MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY_PASSWORD }} # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive # - name: Update dependency graph diff --git a/pom.xml b/pom.xml index e78d907..4ad451c 100644 --- a/pom.xml +++ b/pom.xml @@ -62,18 +62,6 @@ - - - ossrh-snapshot - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - https://github.com/baidu/highflip scm:git:git://github.com/baidu/highflip.git @@ -237,6 +225,18 @@ release + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + @@ -307,23 +307,13 @@ ${nexus.staging.maven.plugin.version} true - sonatype-nexus-repository + ossrh https://oss.sonatype.org true - - - sonatype-nexus-repository - https://oss.sonatype.org/content/repositories/snapshots - - - sonatype-nexus-repository - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - \ No newline at end of file From d3306df671c18e3325380cee622eec640e0173d7 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 3 Aug 2023 16:33:22 +0800 Subject: [PATCH 12/17] add distributionManagement --- highflip-build/pom.xml | 2 +- highflip-clients/highflip-console/pom.xml | 19 ++++++++++--- highflip-clients/highflip-sdk/pom.xml | 19 ++++++++++--- highflip-clients/pom.xml | 4 +-- highflip-core/pom.xml | 19 ++++++++++--- highflip-doc/pom.xml | 2 +- highflip-editor/pom.xml | 6 ++--- highflip-proto/pom.xml | 15 ++++++++++- highflip-server/pom.xml | 6 ++--- .../highflip-adaptor-demo/pom.xml | 4 +-- .../highflip-adaptor-fate/pom.xml | 4 +-- highflip-vendors/pom.xml | 2 +- pom.xml | 27 ++++++++++--------- 13 files changed, 91 insertions(+), 38 deletions(-) diff --git a/highflip-build/pom.xml b/highflip-build/pom.xml index 6d5cf4e..5ae3101 100644 --- a/highflip-build/pom.xml +++ b/highflip-build/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-build jar 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-console/pom.xml b/highflip-clients/highflip-console/pom.xml index 3047fff..0831273 100644 --- a/highflip-clients/highflip-console/pom.xml +++ b/highflip-clients/highflip-console/pom.xml @@ -11,18 +11,31 @@ --> - com.baidu + com.baidu.highflip highflip-console 1.0.0-SNAPSHOT highflip-console - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../../pom.xml + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + 11 11 @@ -44,7 +57,7 @@ spring-boot-starter-json - com.baidu + com.baidu.highflip highflip-sdk 1.0.0-SNAPSHOT diff --git a/highflip-clients/highflip-sdk/pom.xml b/highflip-clients/highflip-sdk/pom.xml index b44148a..fb6434c 100644 --- a/highflip-clients/highflip-sdk/pom.xml +++ b/highflip-clients/highflip-sdk/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-sdk jar 1.0.0-SNAPSHOT @@ -10,15 +10,28 @@ https://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../../pom.xml + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + - com.baidu + com.baidu.highflip highflip-proto ${project.version} diff --git a/highflip-clients/pom.xml b/highflip-clients/pom.xml index 3de5034..1457d0f 100644 --- a/highflip-clients/pom.xml +++ b/highflip-clients/pom.xml @@ -4,13 +4,13 @@ highflip - com.baidu + com.baidu.highflip 1.0.0-SNAPSHOT ../pom.xml - com.baidu + com.baidu.highflip highflip-clients 1.0.0-SNAPSHOT pom diff --git a/highflip-core/pom.xml b/highflip-core/pom.xml index ff39144..2066608 100644 --- a/highflip-core/pom.xml +++ b/highflip-core/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-core jar 1.0.0-SNAPSHOT @@ -11,15 +11,28 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + - com.baidu + com.baidu.highflip highflip-proto 1.0.0-SNAPSHOT diff --git a/highflip-doc/pom.xml b/highflip-doc/pom.xml index 0d8e57a..3ace508 100644 --- a/highflip-doc/pom.xml +++ b/highflip-doc/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-doc pom 1.0.0-SNAPSHOT diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index 9919f91..d645eb0 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> highflip - com.baidu + com.baidu.highflip 1.0.0-SNAPSHOT 4.0.0 @@ -22,12 +22,12 @@ - com.baidu + com.baidu.highflip highflip-sdk ${project.version} - com.baidu + com.baidu.highflip highflip-proto ${project.version} diff --git a/highflip-proto/pom.xml b/highflip-proto/pom.xml index 1584c1c..868c091 100644 --- a/highflip-proto/pom.xml +++ b/highflip-proto/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-proto jar 1.0.0-SNAPSHOT @@ -16,6 +16,19 @@ 1.49.0 + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + io.grpc diff --git a/highflip-server/pom.xml b/highflip-server/pom.xml index 5055c3e..104b332 100644 --- a/highflip-server/pom.xml +++ b/highflip-server/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-server jar 1.0.0-SNAPSHOT @@ -11,7 +11,7 @@ http://maven.apache.org - com.baidu + com.baidu.highflip highflip 1.0.0-SNAPSHOT ../pom.xml @@ -70,7 +70,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT diff --git a/highflip-vendors/highflip-adaptor-demo/pom.xml b/highflip-vendors/highflip-adaptor-demo/pom.xml index a022666..871c819 100644 --- a/highflip-vendors/highflip-adaptor-demo/pom.xml +++ b/highflip-vendors/highflip-adaptor-demo/pom.xml @@ -3,7 +3,7 @@ 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-demo jar 1.0.0-SNAPSHOT @@ -18,7 +18,7 @@ - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT compile diff --git a/highflip-vendors/highflip-adaptor-fate/pom.xml b/highflip-vendors/highflip-adaptor-fate/pom.xml index 481c757..3fbcc3d 100644 --- a/highflip-vendors/highflip-adaptor-fate/pom.xml +++ b/highflip-vendors/highflip-adaptor-fate/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-adaptor-fate jar 1.0.0-SNAPSHOT @@ -30,7 +30,7 @@ test - com.baidu + com.baidu.highflip highflip-core 1.0.0-SNAPSHOT compile diff --git a/highflip-vendors/pom.xml b/highflip-vendors/pom.xml index 35c88ca..38a435b 100644 --- a/highflip-vendors/pom.xml +++ b/highflip-vendors/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip-vendors pom 1.0.0-SNAPSHOT diff --git a/pom.xml b/pom.xml index 4ad451c..24d99cd 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 - com.baidu + com.baidu.highflip highflip pom 1.0.0-SNAPSHOT @@ -215,6 +215,19 @@ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/content/repositories/snapshots + + + local @@ -225,18 +238,6 @@ release - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/content/repositories/snapshots - - From 446a2f92bfd3a2cf648faa2aafd08b63f1a3ebfd Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Thu, 3 Aug 2023 16:46:04 +0800 Subject: [PATCH 13/17] manual run github action --- .github/workflows/maven.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9dd5638..a766400 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -14,10 +14,6 @@ on: pull_request: branches: [ "master" ] workflow_dispatch: - inputs: - branch: - description: 'The branch to build' - required: true jobs: build: From 545362886dd7659eb5d8fea9c14ef2609b5c6247 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Fri, 4 Aug 2023 16:01:43 +0800 Subject: [PATCH 14/17] modfiy maven deploy config --- .github/workflows/maven.yml | 18 +++++++----------- highflip-clients/highflip-console/pom.xml | 17 ++--------------- highflip-clients/highflip-sdk/pom.xml | 17 ++--------------- highflip-core/pom.xml | 13 ------------- highflip-editor/pom.xml | 11 ----------- pom.xml | 6 +++--- 6 files changed, 14 insertions(+), 68 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a766400..345d0a8 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -29,11 +29,11 @@ jobs: with: java-version: '11' distribution: 'temurin' -# cache: maven -# server-username: MAVEN_USERNAME -# server-password: MAVEN_PASSWORD -# gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} -# gpg-passphrase: MAVEN_GPG_PASSPHRASE + server-id: ossrh + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Build with Maven run: mvn -B package -DskipTests -Prelease --file pom.xml @@ -51,12 +51,8 @@ jobs: - name: Publish to the Maven Central Repository run: | - mvn clean \ - --no-transfer-progress \ - --batch-mode \ - -Prelease \ - -DskipTests \ - deploy --file pom.xml + mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests \ + deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am env: MAVEN_USERNAME: ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} MAVEN_PASSWORD: ${{ secrets.HIGHFLIP_MAVEN_TOKEN }} diff --git a/highflip-clients/highflip-console/pom.xml b/highflip-clients/highflip-console/pom.xml index 0831273..1815e6d 100644 --- a/highflip-clients/highflip-console/pom.xml +++ b/highflip-clients/highflip-console/pom.xml @@ -18,24 +18,11 @@ com.baidu.highflip - highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/content/repositories/snapshots - - - 11 11 diff --git a/highflip-clients/highflip-sdk/pom.xml b/highflip-clients/highflip-sdk/pom.xml index fb6434c..49e9514 100644 --- a/highflip-clients/highflip-sdk/pom.xml +++ b/highflip-clients/highflip-sdk/pom.xml @@ -11,24 +11,11 @@ com.baidu.highflip - highflip + highflip-clients 1.0.0-SNAPSHOT - ../../pom.xml + ../pom.xml - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/content/repositories/snapshots - - - com.baidu.highflip diff --git a/highflip-core/pom.xml b/highflip-core/pom.xml index 2066608..22b019e 100644 --- a/highflip-core/pom.xml +++ b/highflip-core/pom.xml @@ -17,19 +17,6 @@ ../pom.xml - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/content/repositories/snapshots - - - com.baidu.highflip diff --git a/highflip-editor/pom.xml b/highflip-editor/pom.xml index d645eb0..57aa44c 100644 --- a/highflip-editor/pom.xml +++ b/highflip-editor/pom.xml @@ -11,15 +11,6 @@ highflip-editor - - - - - - - - - com.baidu.highflip @@ -103,6 +94,4 @@ - - \ No newline at end of file diff --git a/pom.xml b/pom.xml index 24d99cd..5358d7a 100644 --- a/pom.xml +++ b/pom.xml @@ -63,9 +63,9 @@ - https://github.com/baidu/highflip - scm:git:git://github.com/baidu/highflip.git - scm:git:ssh://git@github.com/baidu/highflip.git + scm:git:${project.scm.url} + scm:git:${project.scm.url} + https://github.com/baidu/highflip.git HEAD From ffcd7d8987f73e9c6a696a07adb1b17a18b51519 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Fri, 4 Aug 2023 16:48:20 +0800 Subject: [PATCH 15/17] test secrets --- .github/workflows/maven.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 345d0a8..2005176 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -51,6 +51,8 @@ jobs: - name: Publish to the Maven Central Repository run: | + echo "this is a test: " + echo ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} \ mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests \ deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am env: From 3c0a4a8d9db51fd04c9469350df2ca2443cd4cf4 Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Fri, 4 Aug 2023 16:56:59 +0800 Subject: [PATCH 16/17] test maven.yaml --- .github/workflows/maven.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 2005176..ef60c26 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -50,9 +50,11 @@ jobs: gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase - name: Publish to the Maven Central Repository + if: env.MAVEN_USERNAME == 'chenzhiyu' run: | - echo "this is a test: " - echo ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} \ + echo "i know, this is a test: " + echo ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} + echo "test over" mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests \ deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am env: From 26694bd6c8922ab9d28a85fe42e8ad6ca46c6c9b Mon Sep 17 00:00:00 2001 From: TingSty <615874106@qq.com> Date: Fri, 4 Aug 2023 17:33:55 +0800 Subject: [PATCH 17/17] echo github context --- .github/workflows/maven.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index ef60c26..90b2353 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -49,12 +49,13 @@ jobs: gpg-private-key: ${{ secrets.HIGHFLIP_GPG_SECRET_KEY }} # Value of the GPG private key to import gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Publish to the Maven Central Repository - if: env.MAVEN_USERNAME == 'chenzhiyu' - run: | - echo "i know, this is a test: " - echo ${{ secrets.HIGHFLIP_MAVEN_USERNAME }} - echo "test over" + run: mvn clean --no-transfer-progress --batch-mode -Prelease -DskipTests \ deploy --file pom.xml -pl highflip-proto,highflip-core,highflip-clients/highflip-sdk -am env: