geg-gas/powerjob-kingbase
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

---初始化项目

Browse Source
This commit is contained in:
张福财
2025-09-19 16:14:08 +08:00
parent 902d3d7e3b
commit afee7c03ac
767 changed files with 75809 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
powerjob-server/powerjob-server-remote/pom.xml Normal file
View File

@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>powerjob-server</artifactId>
<groupId>tech.powerjob</groupId>
<version>5.1.2</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>powerjob-server-remote</artifactId>
<version>${project.parent.version}</version>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>tech.powerjob</groupId>
<artifactId>powerjob-server-common</artifactId>
</dependency>
<dependency>
<groupId>tech.powerjob</groupId>
<artifactId>powerjob-server-extension</artifactId>
</dependency>
<dependency>
<groupId>tech.powerjob</groupId>
<artifactId>powerjob-server-persistence</artifactId>
</dependency>
</dependencies>
</project>

15
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/aware/TransportServiceAware.java Normal file
View File

@ -0,0 +1,15 @@
package tech.powerjob.server.remote.aware;
import tech.powerjob.server.common.aware.PowerJobAware;
import tech.powerjob.server.remote.transporter.TransportService;
/**
* TransportServiceAware
*
* @author tjq
* @since 2023/3/4
*/
public interface TransportServiceAware extends PowerJobAware {
void setTransportService(TransportService transportService);
}

59
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/FriendActor.java Normal file
View File

@ -0,0 +1,59 @@
package tech.powerjob.server.remote.server;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.stereotype.Component;
import tech.powerjob.common.response.AskResponse;
import tech.powerjob.common.serialize.JsonUtils;
import tech.powerjob.remote.framework.actor.Actor;
import tech.powerjob.remote.framework.actor.Handler;
import tech.powerjob.remote.framework.actor.ProcessType;
import tech.powerjob.server.remote.aware.TransportServiceAware;
import tech.powerjob.server.remote.server.election.Ping;
import tech.powerjob.server.remote.server.redirector.RemoteProcessReq;
import tech.powerjob.server.remote.server.redirector.RemoteRequestProcessor;
import tech.powerjob.server.remote.transporter.TransportService;
import static tech.powerjob.common.RemoteConstant.*;
/**
* 处理朋友们的信息（处理服务器与服务器之间的通讯）
*
* @author tjq
* @since 2020/4/9
*/
@Slf4j
@Component
@Actor(path = S4S_PATH)
public class FriendActor implements TransportServiceAware {
private TransportService transportService;
/**
* 处理存活检测的请求
*/
@Handler(path = S4S_HANDLER_PING, processType = ProcessType.NO_BLOCKING)
public AskResponse onReceivePing(Ping ping) {
return AskResponse.succeed(transportService.allProtocols());
}
@Handler(path = S4S_HANDLER_PROCESS, processType = ProcessType.BLOCKING)
public AskResponse onReceiveRemoteProcessReq(RemoteProcessReq req) {
AskResponse response = new AskResponse();
response.setSuccess(true);
try {
response.setData(JsonUtils.toBytes(RemoteRequestProcessor.processRemoteRequest(req)));
} catch (Throwable t) {
log.error("[FriendActor] process remote request[{}] failed!", req, t);
response.setSuccess(false);
response.setMessage(ExceptionUtils.getMessage(t));
}
return response;
}
@Override
public void setTransportService(TransportService transportService) {
this.transportService = transportService;
}
}

16
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/Ping.java Normal file
View File

@ -0,0 +1,16 @@
package tech.powerjob.server.remote.server.election;
import tech.powerjob.common.PowerSerializable;
import lombok.Data;
/**
* 检测目标机器是否存活
*
* @author tjq
* @since 2020/4/5
*/
@Data
public class Ping implements PowerSerializable {
private long currentTime;
}

180
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/election/ServerElectionService.java Normal file
View File

@ -0,0 +1,180 @@
package tech.powerjob.server.remote.server.election;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import tech.powerjob.common.exception.PowerJobException;
import tech.powerjob.common.request.ServerDiscoveryRequest;
import tech.powerjob.common.response.AskResponse;
import tech.powerjob.common.serialize.JsonUtils;
import tech.powerjob.remote.framework.base.URL;
import tech.powerjob.server.extension.LockService;
import tech.powerjob.server.persistence.remote.model.AppInfoDO;
import tech.powerjob.server.persistence.remote.repository.AppInfoRepository;
import tech.powerjob.server.remote.transporter.ProtocolInfo;
import tech.powerjob.server.remote.transporter.TransportService;
import tech.powerjob.server.remote.transporter.impl.ServerURLFactory;
import java.util.Date;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
/**
* Default server election policy, first-come, first-served, no load balancing capability
*
* @author tjq
* @since 2021/2/9
*/
@Slf4j
@Service
public class ServerElectionService {
private final LockService lockService;
private final TransportService transportService;
private final AppInfoRepository appInfoRepository;
private final int accurateSelectServerPercentage;
private static final int RETRY_TIMES = 10;
private static final long PING_TIMEOUT_MS = 1000;
private static final String SERVER_ELECT_LOCK = "server_elect_%d";
public ServerElectionService(LockService lockService, TransportService transportService, AppInfoRepository appInfoRepository,@Value("${oms.accurate.select.server.percentage}") int accurateSelectServerPercentage) {
this.lockService = lockService;
this.transportService = transportService;
this.appInfoRepository = appInfoRepository;
this.accurateSelectServerPercentage = accurateSelectServerPercentage;
}
public String elect(ServerDiscoveryRequest request) {
if (!accurate()) {
final String currentServer = request.getCurrentServer();
// 如果是本机，就不需要查数据库那么复杂的操作了，直接返回成功
Optional<ProtocolInfo> localProtocolInfoOpt = Optional.ofNullable(transportService.allProtocols().get(request.getProtocol()));
if (localProtocolInfoOpt.isPresent()) {
if (localProtocolInfoOpt.get().getExternalAddress().equals(currentServer) || localProtocolInfoOpt.get().getAddress().equals(currentServer)) {
log.info("[ServerElection] this server[{}] is worker[appId={}]'s current server, skip check", currentServer, request.getAppId());
return currentServer;
}
}
}
return getServer0(request);
}
private String getServer0(ServerDiscoveryRequest discoveryRequest) {
final Long appId = discoveryRequest.getAppId();
final String protocol = discoveryRequest.getProtocol();
Set<String> downServerCache = Sets.newHashSet();
for (int i = 0; i < RETRY_TIMES; i++) {
// 无锁获取当前数据库中的Server
Optional<AppInfoDO> appInfoOpt = appInfoRepository.findById(appId);
if (!appInfoOpt.isPresent()) {
throw new PowerJobException(appId + " is not registered!");
}
String appName = appInfoOpt.get().getAppName();
String originServer = appInfoOpt.get().getCurrentServer();
String activeAddress = activeAddress(originServer, downServerCache, protocol);
if (StringUtils.isNotEmpty(activeAddress)) {
return activeAddress;
}
// 无可用Server，重新进行Server选举，需要加锁
String lockName = String.format(SERVER_ELECT_LOCK, appId);
boolean lockStatus = lockService.tryLock(lockName, 30000);
if (!lockStatus) {
try {
Thread.sleep(500);
}catch (Exception ignore) {
}
continue;
}
try {
// 可能上一台机器已经完成了Server选举，需要再次判断
AppInfoDO appInfo = appInfoRepository.findById(appId).orElseThrow(() -> new RuntimeException("impossible, unless we just lost our database."));
String address = activeAddress(appInfo.getCurrentServer(), downServerCache, protocol);
if (StringUtils.isNotEmpty(address)) {
return address;
}
// 篡位，如果本机存在协议，则作为Server调度该 worker
final ProtocolInfo targetProtocolInfo = transportService.allProtocols().get(protocol);
if (targetProtocolInfo != null) {
// 注意，写入 AppInfoDO#currentServer 的永远是 default 的绑定地址，仅在返回的时候特殊处理为协议地址
appInfo.setCurrentServer(transportService.defaultProtocol().getAddress());
appInfo.setGmtModified(new Date());
appInfoRepository.saveAndFlush(appInfo);
log.info("[ServerElection] this server({}) become the new server for app(appId={}).", appInfo.getCurrentServer(), appId);
return targetProtocolInfo.getExternalAddress();
}
}catch (Exception e) {
log.error("[ServerElection] write new server to db failed for app {}.", appName, e);
} finally {
lockService.unlock(lockName);
}
}
throw new PowerJobException("server elect failed for app " + appId);
}
/**
* 判断指定server是否存活
* @param serverAddress 需要检测的server地址（绑定的内网地址）
* @param downServerCache 缓存，防止多次发送PING（这个QPS其实还蛮爆表的...）
* @param protocol 协议，用于返回指定的地址
* @return null or address（外部地址）
*/
private String activeAddress(String serverAddress, Set<String> downServerCache, String protocol) {
if (downServerCache.contains(serverAddress)) {
return null;
}
if (StringUtils.isEmpty(serverAddress)) {
return null;
}
Ping ping = new Ping();
ping.setCurrentTime(System.currentTimeMillis());
URL targetUrl = ServerURLFactory.ping2Friend(serverAddress);
try {
AskResponse response = transportService.ask(transportService.defaultProtocol().getProtocol(), targetUrl, ping, AskResponse.class)
.toCompletableFuture()
.get(PING_TIMEOUT_MS, TimeUnit.MILLISECONDS);
if (response.isSuccess()) {
// 检测通过的是远程 server 的暴露地址，需要返回 worker 需要的协议地址
final JSONObject protocolInfo = JsonUtils.parseObject(response.getData(), JSONObject.class).getJSONObject(protocol);
if (protocolInfo != null) {
downServerCache.remove(serverAddress);
ProtocolInfo remoteProtocol = protocolInfo.toJavaObject(ProtocolInfo.class);
log.info("[ServerElection] server[{}] is active, it will be the master, final protocol={}", serverAddress, remoteProtocol);
// 4.3.3 升级 4.3.4 过程中，未升级的 server 还不存在 externalAddress，需要使用 address 兼容
return Optional.ofNullable(remoteProtocol.getExternalAddress()).orElse(remoteProtocol.getAddress());
} else {
log.warn("[ServerElection] server[{}] is active but don't have target protocol", serverAddress);
}
}
} catch (TimeoutException te) {
log.warn("[ServerElection] server[{}] was down due to ping timeout!", serverAddress);
} catch (Exception e) {
log.warn("[ServerElection] server[{}] was down with unknown case!", serverAddress, e);
}
downServerCache.add(serverAddress);
return null;
}
private boolean accurate() {
return ThreadLocalRandom.current().nextInt(100) < accurateSelectServerPercentage;
}
}

24
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/redirector/DesignateServer.java Normal file
View File

@ -0,0 +1,24 @@
package tech.powerjob.server.remote.server.redirector;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
* 需要在指定的服务器运行
* 注意：该注解所在方法的参数必须为对象，不可以是 long 等基本类型
*
* @author tjq
* @since 12/13/20
*/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface DesignateServer {
/**
* 转发请求需要 AppInfo 下的 currentServer 信息，因此必须要有 appId 作为入参，该字段指定了 appId 字段的参数名称，默认为 appId
* @return appId 参数名称
*/
String appIdParameterName() default "appId";
}

136
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/redirector/DesignateServerAspect.java Normal file
View File

@ -0,0 +1,136 @@
package tech.powerjob.server.remote.server.redirector;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.TypeFactory;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.Signature;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import tech.powerjob.common.RemoteConstant;
import tech.powerjob.common.exception.PowerJobException;
import tech.powerjob.common.response.AskResponse;
import tech.powerjob.remote.framework.base.URL;
import tech.powerjob.server.persistence.remote.model.AppInfoDO;
import tech.powerjob.server.persistence.remote.repository.AppInfoRepository;
import tech.powerjob.server.remote.transporter.TransportService;
import tech.powerjob.server.remote.transporter.impl.ServerURLFactory;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.TimeUnit;
/**
* 指定服务器运行切面
*
* @author tjq
* @since 12/13/20
*/
@Slf4j
@Aspect
@Component
@Order(0)
@RequiredArgsConstructor
public class DesignateServerAspect {
private final TransportService transportService;
private final AppInfoRepository appInfoRepository;
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@Around(value = "@annotation(designateServer))")
public Object execute(ProceedingJoinPoint point, DesignateServer designateServer) throws Throwable {
// 参数
Object[] args = point.getArgs();
// 方法名
String methodName = point.getSignature().getName();
// 类名
String className = point.getSignature().getDeclaringTypeName();
Signature signature = point.getSignature();
// 方法签名
MethodSignature methodSignature = (MethodSignature) signature;
String[] parameterNames = methodSignature.getParameterNames();
String[] parameterTypes = Arrays.stream(methodSignature.getParameterTypes()).map(Class::getName).toArray(String[]::new);
Long appId = null;
for (int i = 0; i < parameterNames.length; i++) {
if (StringUtils.equals(parameterNames[i], designateServer.appIdParameterName())) {
appId = Long.parseLong(String.valueOf(args[i]));
break;
}
}
if (appId == null) {
throw new PowerJobException("can't find appId in params for:" + signature);
}
// 获取执行机器
AppInfoDO appInfo = appInfoRepository.findById(appId).orElseThrow(() -> new PowerJobException("can't find app info"));
String targetServer = appInfo.getCurrentServer();
// 目标IP为空，本地执行
if (StringUtils.isEmpty(targetServer)) {
return point.proceed();
}
// 目标IP与本地符合则本地执行
if (Objects.equals(targetServer, transportService.defaultProtocol().getAddress())) {
return point.proceed();
}
log.info("[DesignateServerAspect] the method[{}] should execute in server[{}], so this request will be redirect to remote server!", signature.toShortString(), targetServer);
// 转发请求，远程执行后返回结果
RemoteProcessReq remoteProcessReq = new RemoteProcessReq()
.setClassName(className)
.setMethodName(methodName)
.setParameterTypes(parameterTypes)
.setArgs(args);
final URL friendUrl = ServerURLFactory.process2Friend(targetServer);
CompletionStage<AskResponse> askCS = transportService.ask(transportService.defaultProtocol().getProtocol(), friendUrl, remoteProcessReq, AskResponse.class);
AskResponse askResponse = askCS.toCompletableFuture().get(RemoteConstant.DEFAULT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
if (!askResponse.isSuccess()) {
throw new PowerJobException("remote process failed: " + askResponse.getMessage());
}
// 考虑范型情况
Method method = methodSignature.getMethod();
JavaType returnType = getMethodReturnJavaType(method);
return OBJECT_MAPPER.readValue(askResponse.getData(), returnType);
}
private static JavaType getMethodReturnJavaType(Method method) {
Type type = method.getGenericReturnType();
return getJavaType(type);
}
private static JavaType getJavaType(Type type) {
if (type instanceof ParameterizedType) {
Type[] actualTypeArguments = ((ParameterizedType)type).getActualTypeArguments();
Class<?> rowClass = (Class<?>) ((ParameterizedType)type).getRawType();
JavaType[] javaTypes = new JavaType[actualTypeArguments.length];
for (int i = 0; i < actualTypeArguments.length; i++) {
//泛型也可能带有泛型，递归处理
javaTypes[i] = getJavaType(actualTypeArguments[i]);
}
return TypeFactory.defaultInstance().constructParametricType(rowClass, javaTypes);
} else {
return TypeFactory.defaultInstance().constructParametricType((Class<?>) type, new JavaType[0]);
}
}
}

25
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/redirector/RemoteProcessReq.java Normal file
View File

@ -0,0 +1,25 @@
package tech.powerjob.server.remote.server.redirector;
import tech.powerjob.common.PowerSerializable;
import lombok.Getter;
import lombok.Setter;
import lombok.experimental.Accessors;
/**
* 原创执行命令
*
* @author tjq
* @since 12/13/20
*/
@Getter
@Setter
@Accessors(chain = true)
public class RemoteProcessReq implements PowerSerializable {
private String className;
private String methodName;
private String[] parameterTypes;
private Object[] args;
}

38
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/redirector/RemoteRequestProcessor.java Normal file
View File

@ -0,0 +1,38 @@
package tech.powerjob.server.remote.server.redirector;
import com.alibaba.fastjson.JSONObject;
import tech.powerjob.server.common.utils.SpringUtils;
import org.springframework.util.ReflectionUtils;
import java.lang.reflect.Method;
/**
* process remote request
*
* @author tjq
* @since 2021/2/19
*/
public class RemoteRequestProcessor {
public static Object processRemoteRequest(RemoteProcessReq req) throws ClassNotFoundException {
Object[] args = req.getArgs();
String[] parameterTypes = req.getParameterTypes();
Class<?>[] parameters = new Class[parameterTypes.length];
for (int i = 0; i < parameterTypes.length; i++) {
parameters[i] = Class.forName(parameterTypes[i]);
Object arg = args[i];
if (arg != null) {
args[i] = JSONObject.parseObject(JSONObject.toJSONBytes(arg), parameters[i]);
}
}
Class<?> clz = Class.forName(req.getClassName());
Object bean = SpringUtils.getBean(clz);
Method method = ReflectionUtils.findMethod(clz, req.getMethodName(), parameters);
assert method != null;
return ReflectionUtils.invokeMethod(method, bean, args);
}
}

26
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoService.java Normal file
View File

@ -0,0 +1,26 @@
package tech.powerjob.server.remote.server.self;
import tech.powerjob.server.common.module.ServerInfo;
/**
* ServerInfoService
*
* @author tjq
* @since 2022/9/12
*/
public interface ServerInfoService {
/**
* fetch current server info
* @return ServerInfo
*/
ServerInfo fetchCurrentServerInfo();
/**
* fetch schedule server info
* @param appId appId
* @return ServerInfo
*/
ServerInfo fetchAppServerInfo(Long appId);
}

151
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/server/self/ServerInfoServiceImpl.java Normal file
View File

@ -0,0 +1,151 @@
package tech.powerjob.server.remote.server.self;
import com.google.common.base.Stopwatch;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.info.BuildProperties;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import tech.powerjob.common.exception.PowerJobException;
import tech.powerjob.common.utils.CommonUtils;
import tech.powerjob.common.utils.NetUtils;
import tech.powerjob.server.common.module.ServerInfo;
import tech.powerjob.server.extension.LockService;
import tech.powerjob.server.persistence.remote.model.ServerInfoDO;
import tech.powerjob.server.persistence.remote.repository.ServerInfoRepository;
import tech.powerjob.server.remote.server.redirector.DesignateServer;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* management server info, like heartbeat, server id etc
*
* @author tjq
* @since 2021/2/21
*/
@Slf4j
@Service
public class ServerInfoServiceImpl implements ServerInfoService {
private final ServerInfo serverInfo;
private final ServerInfoRepository serverInfoRepository;
private static final long MAX_SERVER_CLUSTER_SIZE = 10000;
private static final String SERVER_INIT_LOCK = "server_init_lock";
private static final int SERVER_INIT_LOCK_MAX_TIME = 15000;
@Autowired
public ServerInfoServiceImpl(LockService lockService, ServerInfoRepository serverInfoRepository) {
this.serverInfo = new ServerInfo();
String ip = NetUtils.getLocalHost();
serverInfo.setIp(ip);
serverInfo.setBornTime(System.currentTimeMillis());
this.serverInfoRepository = serverInfoRepository;
Stopwatch sw = Stopwatch.createStarted();
while (!lockService.tryLock(SERVER_INIT_LOCK, SERVER_INIT_LOCK_MAX_TIME)) {
log.info("[ServerInfoService] waiting for lock: {}", SERVER_INIT_LOCK);
CommonUtils.easySleep(100);
}
try {
// register server then get server_id
ServerInfoDO server = serverInfoRepository.findByIp(ip);
if (server == null) {
ServerInfoDO newServerInfo = new ServerInfoDO(ip);
server = serverInfoRepository.saveAndFlush(newServerInfo);
} else {
serverInfoRepository.updateGmtModifiedByIp(ip, new Date());
}
if (server.getId() < MAX_SERVER_CLUSTER_SIZE) {
serverInfo.setId(server.getId());
} else {
long retryServerId = retryServerId();
serverInfo.setId(retryServerId);
serverInfoRepository.updateIdByIp(retryServerId, ip);
}
} catch (Exception e) {
log.error("[ServerInfoService] init server failed", e);
throw e;
} finally {
lockService.unlock(SERVER_INIT_LOCK);
}
log.info("[ServerInfoService] ip:{}, id:{}, cost:{}", ip, serverInfo.getId(), sw);
}
@Scheduled(fixedRate = 15000, initialDelay = 15000)
public void heartbeat() {
serverInfoRepository.updateGmtModifiedByIp(serverInfo.getIp(), new Date());
}
private long retryServerId() {
List<ServerInfoDO> serverInfoList = serverInfoRepository.findAll();
log.info("[ServerInfoService] current server record num in database: {}", serverInfoList.size());
// clean inactive server record first
if (serverInfoList.size() > MAX_SERVER_CLUSTER_SIZE) {
// use a large time interval to prevent valid records from being deleted when the local time is inaccurate
Date oneDayAgo = DateUtils.addDays(new Date(), -1);
int delNum =serverInfoRepository.deleteByGmtModifiedBefore(oneDayAgo);
log.warn("[ServerInfoService] delete invalid {} server info record before {}", delNum, oneDayAgo);
serverInfoList = serverInfoRepository.findAll();
}
if (serverInfoList.size() > MAX_SERVER_CLUSTER_SIZE) {
throw new PowerJobException(String.format("The powerjob-server cluster cannot accommodate %d machines, please rebuild another cluster", serverInfoList.size()));
}
Set<Long> uedServerIds = serverInfoList.stream().map(ServerInfoDO::getId).collect(Collectors.toSet());
for (long i = 1; i <= MAX_SERVER_CLUSTER_SIZE; i++) {
if (uedServerIds.contains(i)) {
continue;
}
log.info("[ServerInfoService] ID[{}] is not used yet, try as new server id", i);
return i;
}
throw new PowerJobException("impossible");
}
@Autowired(required = false)
public void setBuildProperties(BuildProperties buildProperties) {
if (buildProperties == null) {
return;
}
String pomVersion = buildProperties.getVersion();
if (StringUtils.isNotBlank(pomVersion)) {
serverInfo.setVersion(pomVersion);
}
}
@Override
public ServerInfo fetchCurrentServerInfo() {
return serverInfo;
}
@Override
@DesignateServer
public ServerInfo fetchAppServerInfo(Long appId) {
return serverInfo;
}
}

54
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/ProtocolInfo.java Normal file
View File

@ -0,0 +1,54 @@
package tech.powerjob.server.remote.transporter;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import tech.powerjob.common.PowerJobDKey;
import tech.powerjob.common.utils.PropertyUtils;
import tech.powerjob.remote.framework.base.Address;
import tech.powerjob.remote.framework.transporter.Transporter;
/**
* ProtocolInfo
*
* @author tjq
* @since 2023/1/21
*/
@Getter
@Setter
@ToString
public class ProtocolInfo {
private String protocol;
private String address;
/**
* 外部地址，当存在 NAT 等场景时，需要下发该地址到 worker
*/
private String externalAddress;
private transient Transporter transporter;
/**
* 序列化需要，必须存在无参构造方法！严禁删除
*/
public ProtocolInfo() {
}
public ProtocolInfo(String protocol, String host, int port, Transporter transporter) {
this.protocol = protocol;
this.transporter = transporter;
this.address = Address.toFullAddress(host, port);
// 处理外部地址
String externalAddress = PropertyUtils.readProperty(PowerJobDKey.NT_EXTERNAL_ADDRESS, host);
// 考虑到不同协议 port 理论上不一样，server 需要为每个单独的端口配置映射，规则为 powerjob.network.external.port.${协议}，比如 powerjob.network.external.port.http
String externalPortByProtocolKey = PowerJobDKey.NT_EXTERNAL_PORT.concat(".").concat(protocol.toLowerCase());
// 大部分用户只使用一种协议，在此处做兼容处理降低答疑量和提高易用性（如果用户有多种协议，只有被转发的协议能成功通讯）
String externalPort = PropertyUtils.readProperty(externalPortByProtocolKey, PropertyUtils.readProperty(PowerJobDKey.NT_EXTERNAL_PORT, String.valueOf(port)));
this.externalAddress = Address.toFullAddress(externalAddress, Integer.parseInt(externalPort));
}
}

36
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/TransportService.java Normal file
View File

@ -0,0 +1,36 @@
package tech.powerjob.server.remote.transporter;
import tech.powerjob.common.PowerSerializable;
import tech.powerjob.remote.framework.base.RemotingException;
import tech.powerjob.remote.framework.base.URL;
import java.util.Map;
import java.util.concurrent.CompletionStage;
/**
* server 数据传输服务
*
* @author tjq
* @since 2023/1/21
*/
public interface TransportService {
/**
* 自用地址，用于维护 server -> appId 和 server 间通讯
* 4.3.0 前为 ActorSystem Address（ip:10086）
* 4.3.0 后 PowerJob 将主协议切换为自由协议，默认使用 HTTP address （ip:10010）
* @return 自用地址
*/
ProtocolInfo defaultProtocol();
/**
* 当前支持的全部协议
* @return allProtocols
*/
Map<String, ProtocolInfo> allProtocols();
void tell(String protocol, URL url, PowerSerializable request);
<T> CompletionStage<T> ask(String protocol, URL url, PowerSerializable request, Class<T> clz) throws RemotingException;
}

219
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/impl/PowerTransportService.java Normal file
View File

@ -0,0 +1,219 @@
package tech.powerjob.server.remote.transporter.impl;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import tech.powerjob.common.OmsConstant;
import tech.powerjob.common.PowerSerializable;
import tech.powerjob.common.enums.Protocol;
import tech.powerjob.common.utils.NetUtils;
import tech.powerjob.remote.framework.actor.Actor;
import tech.powerjob.remote.framework.base.Address;
import tech.powerjob.remote.framework.base.RemotingException;
import tech.powerjob.remote.framework.base.ServerType;
import tech.powerjob.remote.framework.base.URL;
import tech.powerjob.remote.framework.engine.EngineConfig;
import tech.powerjob.remote.framework.engine.EngineOutput;
import tech.powerjob.remote.framework.engine.RemoteEngine;
import tech.powerjob.remote.framework.engine.impl.PowerJobRemoteEngine;
import tech.powerjob.server.remote.transporter.ProtocolInfo;
import tech.powerjob.server.remote.transporter.TransportService;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletionStage;
/**
* server 数据传输服务
*
* @author tjq
* @since 2023/1/21
*/
@Slf4j
@Service
public class PowerTransportService implements TransportService, InitializingBean, DisposableBean, ApplicationContextAware {
/**
* server 需要激活的通讯协议，建议激活全部支持的协议
*/
@Value("${oms.transporter.active.protocols}")
private String activeProtocols;
/**
* 主要通讯协议，用于 server 与 server 之间的通讯，用户必须保证该协议可用（端口开放）！
*/
@Value("${oms.transporter.main.protocol}")
private String mainProtocol;
private static final String PROTOCOL_PORT_CONFIG = "oms.%s.port";
private final Environment environment;
private ProtocolInfo defaultProtocol;
private final Map<String, ProtocolInfo> protocolName2Info = Maps.newHashMap();
private final List<RemoteEngine> engines = Lists.newArrayList();
private ApplicationContext applicationContext;
public PowerTransportService(Environment environment) {
this.environment = environment;
}
@Override
public ProtocolInfo defaultProtocol() {
return defaultProtocol;
}
@Override
public Map<String, ProtocolInfo> allProtocols() {
return protocolName2Info;
}
private ProtocolInfo fetchProtocolInfo(String protocol) {
// 兼容老版 worker 未上报 protocol 的情况
protocol = compatibleProtocol(protocol);
final ProtocolInfo protocolInfo = protocolName2Info.get(protocol);
if (protocolInfo == null) {
throw new IllegalArgumentException("can't find Transporter by protocol :" + protocol);
}
return protocolInfo;
}
@Override
public void tell(String protocol, URL url, PowerSerializable request) {
fetchProtocolInfo(protocol).getTransporter().tell(url, request);
}
@Override
public <T> CompletionStage<T> ask(String protocol, URL url, PowerSerializable request, Class<T> clz) throws RemotingException {
return fetchProtocolInfo(protocol).getTransporter().ask(url, request, clz);
}
private void initRemoteFrameWork(String protocol, int port) {
// 从构造器注入改为从 applicationContext 获取来避免循环依赖
final Map<String, Object> beansWithAnnotation = applicationContext.getBeansWithAnnotation(Actor.class);
log.info("[PowerTransportService] [{}] find Actor num={},names={}", protocol, beansWithAnnotation.size(), beansWithAnnotation.keySet());
Address address = new Address()
.setHost(NetUtils.getLocalHost())
.setPort(port);
ProtocolInfo protocolInfo = new ProtocolInfo(protocol, address.getHost(), address.getPort(), null);
EngineConfig engineConfig = new EngineConfig()
.setServerType(ServerType.SERVER)
.setType(protocol.toUpperCase())
.setBindAddress(address)
.setActorList(Lists.newArrayList(beansWithAnnotation.values()));
if (!StringUtils.equalsIgnoreCase(protocolInfo.getExternalAddress(), protocolInfo.getAddress())) {
Address externalAddress = Address.fromIpv4(protocolInfo.getExternalAddress());
engineConfig.setExternalAddress(externalAddress);
log.info("[PowerTransportService] [{}] exist externalAddress: {}", protocol, externalAddress);
}
log.info("[PowerTransportService] [{}] start to initialize RemoteEngine[address={}]", protocol, address);
RemoteEngine re = new PowerJobRemoteEngine();
final EngineOutput engineOutput = re.start(engineConfig);
log.info("[PowerTransportService] [{}] start RemoteEngine[address={}] successfully", protocol, address);
this.engines.add(re);
protocolInfo.setTransporter(engineOutput.getTransporter());
this.protocolName2Info.put(protocol, protocolInfo);
}
@Override
public void afterPropertiesSet() throws Exception {
log.info("[PowerTransportService] start to initialize whole PowerTransportService!");
log.info("[PowerTransportService] activeProtocols: {}", activeProtocols);
if (StringUtils.isEmpty(activeProtocols)) {
throw new IllegalArgumentException("activeProtocols can't be empty!");
}
for (String protocol : activeProtocols.split(OmsConstant.COMMA)) {
try {
final int port = parseProtocolPort(protocol);
initRemoteFrameWork(protocol, port);
} catch (Throwable t) {
log.error("[PowerTransportService] initialize protocol[{}] failed. If you don't need to use this protocol, you can turn it off by 'oms.transporter.active.protocols'", protocol);
ExceptionUtils.rethrow(t);
}
}
choseDefault();
log.info("[PowerTransportService] initialize successfully!");
log.info("[PowerTransportService] ALL_PROTOCOLS: {}", protocolName2Info);
}
/**
* 获取协议端口，考虑兼容性 & 用户仔细扩展的场景，选择动态从 env 获取 port
* @return port
*/
private int parseProtocolPort(String protocol) {
final String key1 = String.format(PROTOCOL_PORT_CONFIG, protocol.toLowerCase());
final String key2 = String.format(PROTOCOL_PORT_CONFIG, protocol.toUpperCase());
String portStr = environment.getProperty(key1);
if (StringUtils.isEmpty(portStr)) {
portStr = environment.getProperty(key2);
}
log.info("[PowerTransportService] fetch port for protocol[{}], key={}, value={}", protocol, key1, portStr);
if (StringUtils.isEmpty(portStr)) {
throw new IllegalArgumentException(String.format("can't find protocol config by key: %s, please check your spring config!", key1));
}
return Integer.parseInt(portStr);
}
private String compatibleProtocol(String p) {
if (p == null) {
return Protocol.AKKA.name();
}
return p;
}
/**
* HTTP 优先，否则默认取第一个协议
*/
private void choseDefault() {
this.defaultProtocol = this.protocolName2Info.get(mainProtocol);
log.info("[PowerTransportService] chose [{}] as the default protocol, make sure this protocol can work!", mainProtocol);
if (this.defaultProtocol == null) {
throw new IllegalArgumentException("can't find default protocol, please check your config!");
}
}
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
}
@Override
public void destroy() throws Exception {
engines.forEach(e -> {
try {
e.close();
} catch (Exception ignore) {
}
});
}
}

52
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/transporter/impl/ServerURLFactory.java Normal file
View File

@ -0,0 +1,52 @@
package tech.powerjob.server.remote.transporter.impl;
import tech.powerjob.remote.framework.base.Address;
import tech.powerjob.remote.framework.base.HandlerLocation;
import tech.powerjob.remote.framework.base.ServerType;
import tech.powerjob.remote.framework.base.URL;
import static tech.powerjob.common.RemoteConstant.*;
/**
* 统一生成地址
*
* @author tjq
* @since 2023/1/21
*/
public class ServerURLFactory {
public static URL dispatchJob2Worker(String address) {
return simileBuild(address, ServerType.WORKER, WORKER_PATH, WTT_HANDLER_RUN_JOB);
}
public static URL stopInstance2Worker(String address) {
return simileBuild(address, ServerType.WORKER, WORKER_PATH, WTT_HANDLER_STOP_INSTANCE);
}
public static URL queryInstance2Worker(String address) {
return simileBuild(address, ServerType.WORKER, WORKER_PATH, WTT_HANDLER_QUERY_INSTANCE_STATUS);
}
public static URL deployContainer2Worker(String address) {
return simileBuild(address, ServerType.WORKER, WORKER_PATH, WORKER_HANDLER_DEPLOY_CONTAINER);
}
public static URL destroyContainer2Worker(String address) {
return simileBuild(address, ServerType.WORKER, WORKER_PATH, WORKER_HANDLER_DESTROY_CONTAINER);
}
public static URL ping2Friend(String address) {
return simileBuild(address, ServerType.SERVER, S4S_PATH, S4S_HANDLER_PING);
}
public static URL process2Friend(String address) {
return simileBuild(address, ServerType.SERVER, S4S_PATH, S4S_HANDLER_PROCESS);
}
public static URL simileBuild(String address, ServerType type, String rootPath, String handlerPath) {
return new URL()
.setServerType(type)
.setAddress(Address.fromIpv4(address))
.setLocation(new HandlerLocation().setRootPath(rootPath).setMethodPath(handlerPath));
}
}

118
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/ClusterStatusHolder.java Normal file
View File

@ -0,0 +1,118 @@
package tech.powerjob.server.remote.worker;
import tech.powerjob.common.model.DeployedContainerInfo;
import tech.powerjob.common.request.WorkerHeartbeat;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.CollectionUtils;
import tech.powerjob.server.common.module.WorkerInfo;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* 管理Worker集群状态
*
* @author tjq
* @since 2020/4/5
*/
@Slf4j
public class ClusterStatusHolder {
/**
* 集群所属的应用名称
*/
private final String appName;
/**
* 集群中所有机器的信息
*/
private final Map<String, WorkerInfo> address2WorkerInfo;
/**
* 集群中所有机器的容器部署状态 containerId -> (workerAddress -> containerInfo)
*/
private Map<Long, Map<String, DeployedContainerInfo>> containerId2Infos;
public ClusterStatusHolder(String appName) {
this.appName = appName;
address2WorkerInfo = Maps.newConcurrentMap();
containerId2Infos = Maps.newConcurrentMap();
}
/**
* 更新 worker 机器的状态
* @param heartbeat 心跳请求
*/
public void updateStatus(WorkerHeartbeat heartbeat) {
String workerAddress = heartbeat.getWorkerAddress();
long heartbeatTime = heartbeat.getHeartbeatTime();
WorkerInfo workerInfo = address2WorkerInfo.computeIfAbsent(workerAddress, ignore -> {
WorkerInfo wf = new WorkerInfo();
wf.refresh(heartbeat);
return wf;
});
long oldHeartbeatTime = workerInfo.getLastActiveWorkerTime();
if (heartbeatTime < oldHeartbeatTime) {
log.warn("[ClusterStatusHolder-{}] receive the expired heartbeat from {}, serverTime: {}, heartTime: {}", appName, heartbeat.getWorkerAddress(), System.currentTimeMillis(), heartbeat.getHeartbeatTime());
return;
}
workerInfo.refresh(heartbeat);
List<DeployedContainerInfo> containerInfos = heartbeat.getContainerInfos();
if (!CollectionUtils.isEmpty(containerInfos)) {
containerInfos.forEach(containerInfo -> {
Map<String, DeployedContainerInfo> infos = containerId2Infos.computeIfAbsent(containerInfo.getContainerId(), ignore -> Maps.newConcurrentMap());
infos.put(workerAddress, containerInfo);
});
}
}
/**
* 获取该集群所有的机器信息
* @return 地址: 机器信息
*/
public Map<String, WorkerInfo> getAllWorkers() {
return address2WorkerInfo;
}
/**
* 获取当前该Worker集群容器的部署情况
* @param containerId 容器ID
* @return 该容器的部署情况
*/
public List<DeployedContainerInfo> getDeployedContainerInfos(Long containerId) {
List<DeployedContainerInfo> res = Lists.newLinkedList();
containerId2Infos.getOrDefault(containerId, Collections.emptyMap()).forEach((address, info) -> {
info.setWorkerAddress(address);
res.add(info);
});
return res;
}
/**
* 释放所有本地存储的容器信息（该操作会导致短暂的 listDeployedContainer 服务不可用）
*/
public void release() {
log.info("[ClusterStatusHolder-{}] clean the containerInfos, listDeployedContainer service may down about 1min~", appName);
// 丢弃原来的所有数据，准备重建
containerId2Infos = Maps.newConcurrentMap();
// 丢弃超时机器的信息
List<String> timeoutAddress = Lists.newLinkedList();
address2WorkerInfo.forEach((addr, workerInfo) -> {
if (workerInfo.timeout()) {
timeoutAddress.add(addr);
}
});
if (!timeoutAddress.isEmpty()) {
log.info("[ClusterStatusHolder-{}] detective timeout workers({}), try to release their infos.", appName, timeoutAddress);
timeoutAddress.forEach(address2WorkerInfo::remove);
}
}
}

58
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterManagerService.java Normal file
View File

@ -0,0 +1,58 @@
package tech.powerjob.server.remote.worker;
import tech.powerjob.common.request.WorkerHeartbeat;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* 管理 worker 集群信息
*
* @author tjq
* @since 2020/4/5
*/
@Slf4j
public class WorkerClusterManagerService {
/**
* 存储Worker健康信息，appId -> ClusterStatusHolder
*/
private static final Map<Long, ClusterStatusHolder> APP_ID_2_CLUSTER_STATUS = Maps.newConcurrentMap();
/**
* 更新状态
* @param heartbeat Worker的心跳包
*/
public static void updateStatus(WorkerHeartbeat heartbeat) {
Long appId = heartbeat.getAppId();
String appName = heartbeat.getAppName();
ClusterStatusHolder clusterStatusHolder = APP_ID_2_CLUSTER_STATUS.computeIfAbsent(appId, ignore -> new ClusterStatusHolder(appName));
clusterStatusHolder.updateStatus(heartbeat);
}
/**
* 清理不需要的worker信息
* @param usingAppIds 需要维护的appId，其余的数据将被删除
*/
public static void clean(List<Long> usingAppIds) {
Set<Long> keys = Sets.newHashSet(usingAppIds);
APP_ID_2_CLUSTER_STATUS.entrySet().removeIf(entry -> !keys.contains(entry.getKey()));
}
/**
* 清理缓存信息，防止 OOM
*/
public static void cleanUp() {
APP_ID_2_CLUSTER_STATUS.values().forEach(ClusterStatusHolder::release);
}
protected static Map<Long, ClusterStatusHolder> getAppId2ClusterStatus() {
return APP_ID_2_CLUSTER_STATUS;
}
}

133
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/WorkerClusterQueryService.java Normal file
View File

@ -0,0 +1,133 @@
package tech.powerjob.server.remote.worker;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import tech.powerjob.common.model.DeployedContainerInfo;
import tech.powerjob.server.common.module.WorkerInfo;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import tech.powerjob.server.remote.server.redirector.DesignateServer;
import tech.powerjob.server.remote.worker.filter.WorkerFilter;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* 获取 worker 集群信息
*
* @author tjq
* @since 2021/2/19
*/
@Slf4j
@Service
public class WorkerClusterQueryService {
private final List<WorkerFilter> workerFilters;
public WorkerClusterQueryService(List<WorkerFilter> workerFilters) {
this.workerFilters = workerFilters;
}
/**
* get worker for job
*
* @param jobInfo job
* @return worker cluster info, sorted by metrics desc
*/
public List<WorkerInfo> geAvailableWorkers(JobInfoDO jobInfo) {
List<WorkerInfo> workers = Lists.newLinkedList(getWorkerInfosByAppId(jobInfo.getAppId()).values());
// 过滤不符合要求的机器
workers.removeIf(workerInfo -> filterWorker(workerInfo, jobInfo));
// 限定集群大小（0代表不限制）
if (!workers.isEmpty() && jobInfo.getMaxWorkerCount() > 0 && workers.size() > jobInfo.getMaxWorkerCount()) {
workers = workers.subList(0, jobInfo.getMaxWorkerCount());
}
return workers;
}
@DesignateServer
public List<WorkerInfo> getAllWorkers(Long appId) {
List<WorkerInfo> workers = Lists.newLinkedList(getWorkerInfosByAppId(appId).values());
workers.sort((o1, o2) -> o2.getSystemMetrics().calculateScore() - o1.getSystemMetrics().calculateScore());
return workers;
}
/**
* get all alive workers
*
* @param appId appId
* @return alive workers
*/
@DesignateServer
public List<WorkerInfo> getAllAliveWorkers(Long appId) {
List<WorkerInfo> workers = Lists.newLinkedList(getWorkerInfosByAppId(appId).values());
workers.removeIf(WorkerInfo::timeout);
return workers;
}
/**
* Gets worker info by address.
*
* @param appId the app id
* @param address the address
* @return the worker info by address
*/
public Optional<WorkerInfo> getWorkerInfoByAddress(Long appId, String address) {
// this may cause NPE while address value is null .
final Map<String, WorkerInfo> workerInfosByAppId = getWorkerInfosByAppId(appId);
//add null check for both workerInfos Map and address
if (null != workerInfosByAppId && null != address) {
return Optional.ofNullable(workerInfosByAppId.get(address));
}
return Optional.empty();
}
public Map<Long, ClusterStatusHolder> getAppId2ClusterStatus() {
return WorkerClusterManagerService.getAppId2ClusterStatus();
}
/**
* 获取某个应用容器的部署情况
*
* @param appId 应用ID
* @param containerId 容器ID
* @return 部署情况
*/
public List<DeployedContainerInfo> getDeployedContainerInfos(Long appId, Long containerId) {
ClusterStatusHolder clusterStatusHolder = getAppId2ClusterStatus().get(appId);
if (clusterStatusHolder == null) {
return Collections.emptyList();
}
return clusterStatusHolder.getDeployedContainerInfos(containerId);
}
private Map<String, WorkerInfo> getWorkerInfosByAppId(Long appId) {
ClusterStatusHolder clusterStatusHolder = getAppId2ClusterStatus().get(appId);
if (clusterStatusHolder == null) {
log.warn("[WorkerManagerService] can't find any worker for app(appId={}) yet.", appId);
return Collections.emptyMap();
}
return clusterStatusHolder.getAllWorkers();
}
/**
* filter invalid worker for job
*
* @param workerInfo worker info
* @param jobInfo job info
* @return filter this worker when return true
*/
private boolean filterWorker(WorkerInfo workerInfo, JobInfoDO jobInfo) {
for (WorkerFilter filter : workerFilters) {
if (filter.filter(workerInfo, jobInfo)) {
return true;
}
}
return false;
}
}

44
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DesignatedWorkerFilter.java Normal file
View File

@ -0,0 +1,44 @@
package tech.powerjob.server.remote.worker.filter;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import tech.powerjob.server.common.SJ;
import tech.powerjob.server.common.module.WorkerInfo;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import java.util.Set;
/**
* just use designated worker
*
* @author tjq
* @since 2021/2/19
*/
@Slf4j
@Component
public class DesignatedWorkerFilter implements WorkerFilter {
@Override
public boolean filter(WorkerInfo workerInfo, JobInfoDO jobInfo) {
String designatedWorkers = jobInfo.getDesignatedWorkers();
// no worker is specified, no filter of any
if (StringUtils.isEmpty(designatedWorkers)) {
return false;
}
Set<String> designatedWorkersSet = Sets.newHashSet(SJ.COMMA_SPLITTER.splitToList(designatedWorkers));
for (String tagOrAddress : designatedWorkersSet) {
if (tagOrAddress.equals(workerInfo.getTag()) || tagOrAddress.equals(workerInfo.getAddress())) {
return false;
}
}
return true;
}
}

26
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/DisconnectedWorkerFilter.java Normal file
View File

@ -0,0 +1,26 @@
package tech.powerjob.server.remote.worker.filter;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import tech.powerjob.server.common.module.WorkerInfo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
/**
* filter disconnected worker
*
* @author tjq
* @since 2021/2/19
*/
@Slf4j
@Component
public class DisconnectedWorkerFilter implements WorkerFilter {
@Override
public boolean filter(WorkerInfo workerInfo, JobInfoDO jobInfo) {
boolean timeout = workerInfo.timeout();
if (timeout) {
log.info("[Job-{}] filter worker[{}] due to timeout(lastActiveTime={},lastActiveTimeWorkerTime={})", jobInfo.getId(), workerInfo.getAddress(), workerInfo.getLastActiveTime(), workerInfo.getLastActiveWorkerTime());
}
return timeout;
}
}

28
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/SystemMetricsWorkerFilter.java Normal file
View File

@ -0,0 +1,28 @@
package tech.powerjob.server.remote.worker.filter;
import tech.powerjob.common.model.SystemMetrics;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import tech.powerjob.server.common.module.WorkerInfo;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
/**
* filter worker by system metric
*
* @author tjq
* @since 2021/2/19
*/
@Slf4j
@Component
public class SystemMetricsWorkerFilter implements WorkerFilter {
@Override
public boolean filter(WorkerInfo workerInfo, JobInfoDO jobInfo) {
SystemMetrics metrics = workerInfo.getSystemMetrics();
boolean filter = !metrics.available(jobInfo.getMinCpuCores(), jobInfo.getMinMemorySpace(), jobInfo.getMinDiskSpace());
if (filter) {
log.info("[Job-{}] filter worker[{}] because the {} do not meet the requirements", jobInfo.getId(), workerInfo.getAddress(), workerInfo.getSystemMetrics());
}
return filter;
}
}

21
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/filter/WorkerFilter.java Normal file
View File

@ -0,0 +1,21 @@
package tech.powerjob.server.remote.worker.filter;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import tech.powerjob.server.common.module.WorkerInfo;
/**
* filter worker by system metrics or other info
*
* @author tjq
* @since 2021/2/16
*/
public interface WorkerFilter {
/**
*
* @param workerInfo worker info, maybe you need to use your customized info in SystemMetrics#extra
* @param jobInfoDO job info
* @return true will remove the worker in process list
*/
boolean filter(WorkerInfo workerInfo, JobInfoDO jobInfoDO);
}

32
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/selector/TaskTrackerSelector.java Normal file
View File

@ -0,0 +1,32 @@
package tech.powerjob.server.remote.worker.selector;
import tech.powerjob.common.enums.DispatchStrategy;
import tech.powerjob.server.common.module.WorkerInfo;
import tech.powerjob.server.persistence.remote.model.InstanceInfoDO;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import java.util.List;
/**
* 主节点选择方式
*
* @author tjq
* @since 2024/2/24
*/
public interface TaskTrackerSelector {
/**
* 支持的策略
* @return 派发策略
*/
DispatchStrategy strategy();
/**
* 选择主节点
* @param jobInfoDO 任务信息
* @param instanceInfoDO 任务实例
* @param availableWorkers 可用 workers
* @return 主节点 worker
*/
WorkerInfo select(JobInfoDO jobInfoDO, InstanceInfoDO instanceInfoDO, List<WorkerInfo> availableWorkers);
}

33
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/selector/TaskTrackerSelectorService.java Normal file
View File

@ -0,0 +1,33 @@
package tech.powerjob.server.remote.worker.selector;
import com.google.common.collect.Maps;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import tech.powerjob.server.common.module.WorkerInfo;
import tech.powerjob.server.persistence.remote.model.InstanceInfoDO;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import java.util.List;
import java.util.Map;
/**
* TaskTrackerSelectorService
*
* @author tjq
* @since 2024/2/24
*/
@Service
public class TaskTrackerSelectorService {
private final Map<Integer, TaskTrackerSelector> taskTrackerSelectorMap = Maps.newHashMap();
@Autowired
public TaskTrackerSelectorService(List<TaskTrackerSelector> taskTrackerSelectors) {
taskTrackerSelectors.forEach(ts -> taskTrackerSelectorMap.put(ts.strategy().getV(), ts));
}
public WorkerInfo select(JobInfoDO jobInfoDO, InstanceInfoDO instanceInfoDO, List<WorkerInfo> availableWorkers) {
TaskTrackerSelector taskTrackerSelector = taskTrackerSelectorMap.get(jobInfoDO.getDispatchStrategy());
return taskTrackerSelector.select(jobInfoDO, instanceInfoDO, availableWorkers);
}
}

33
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/selector/impl/HealthFirstTaskTrackerSelector.java Normal file
View File

@ -0,0 +1,33 @@
package tech.powerjob.server.remote.worker.selector.impl;
import com.google.common.collect.Lists;
import org.springframework.stereotype.Component;
import tech.powerjob.common.enums.DispatchStrategy;
import tech.powerjob.server.common.module.WorkerInfo;
import tech.powerjob.server.persistence.remote.model.InstanceInfoDO;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import tech.powerjob.server.remote.worker.selector.TaskTrackerSelector;
import java.util.List;
/**
* HealthFirst
*
* @author （疑似）新冠帕鲁
* @since 2024/2/24
*/
@Component
public class HealthFirstTaskTrackerSelector implements TaskTrackerSelector {
@Override
public DispatchStrategy strategy() {
return DispatchStrategy.HEALTH_FIRST;
}
@Override
public WorkerInfo select(JobInfoDO jobInfoDO, InstanceInfoDO instanceInfoDO, List<WorkerInfo> availableWorkers) {
List<WorkerInfo> workers = Lists.newArrayList(availableWorkers);
workers.sort((o1, o2) -> o2.getSystemMetrics().calculateScore() - o1.getSystemMetrics().calculateScore());
return workers.get(0);
}
}

32
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/selector/impl/RandomTaskTrackerSelector.java Normal file
View File

@ -0,0 +1,32 @@
package tech.powerjob.server.remote.worker.selector.impl;
import org.springframework.stereotype.Component;
import tech.powerjob.common.enums.DispatchStrategy;
import tech.powerjob.server.common.module.WorkerInfo;
import tech.powerjob.server.persistence.remote.model.InstanceInfoDO;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import tech.powerjob.server.remote.worker.selector.TaskTrackerSelector;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
/**
* RANDOM
*
* @author （疑似）新冠帕鲁
* @since 2024/2/24
*/
@Component
public class RandomTaskTrackerSelector implements TaskTrackerSelector {
@Override
public DispatchStrategy strategy() {
return DispatchStrategy.RANDOM;
}
@Override
public WorkerInfo select(JobInfoDO jobInfoDO, InstanceInfoDO instanceInfoDO, List<WorkerInfo> availableWorkers) {
int randomIdx = ThreadLocalRandom.current().nextInt(availableWorkers.size());
return availableWorkers.get(randomIdx);
}
}

59
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/selector/impl/SpecifyTaskTrackerSelector.java Normal file
View File

@ -0,0 +1,59 @@
package tech.powerjob.server.remote.worker.selector.impl;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import tech.powerjob.common.enums.DispatchStrategy;
import tech.powerjob.common.utils.CollectionUtils;
import tech.powerjob.server.common.module.WorkerInfo;
import tech.powerjob.server.persistence.remote.model.InstanceInfoDO;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import tech.powerjob.server.remote.worker.selector.TaskTrackerSelector;
import tech.powerjob.server.remote.worker.utils.SpecifyUtils;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
/**
* 指定工作的主节点，大规模运算需要隔离主节点，以防止 worker 部署打断整体的任务执行
*
* @author tjq
* @since 2024/2/24
*/
@Slf4j
@Component
public class SpecifyTaskTrackerSelector implements TaskTrackerSelector {
@Override
public DispatchStrategy strategy() {
return DispatchStrategy.SPECIFY;
}
@Override
public WorkerInfo select(JobInfoDO jobInfoDO, InstanceInfoDO instanceInfoDO, List<WorkerInfo> availableWorkers) {
String dispatchStrategyConfig = jobInfoDO.getDispatchStrategyConfig();
// 降级到随机
if (StringUtils.isEmpty(dispatchStrategyConfig)) {
log.warn("[SpecifyTaskTrackerSelector] job[id={}]'s dispatchStrategyConfig is empty, use random as bottom DispatchStrategy!", jobInfoDO.getId());
return availableWorkers.get(ThreadLocalRandom.current().nextInt(availableWorkers.size()));
}
List<WorkerInfo> targetWorkers = Lists.newArrayList();
availableWorkers.forEach(aw -> {
boolean match = SpecifyUtils.match(aw, dispatchStrategyConfig);
if (match) {
targetWorkers.add(aw);
}
});
if (CollectionUtils.isEmpty(targetWorkers)) {
log.warn("[SpecifyTaskTrackerSelector] Unable to find available nodes based on conditions for job(id={},dispatchStrategyConfig={}), use random as bottom DispatchStrategy!", jobInfoDO.getId(), dispatchStrategyConfig);
return availableWorkers.get(ThreadLocalRandom.current().nextInt(availableWorkers.size()));
}
// 如果有多个 worker 符合条件，最终还是随机选择出一个
return targetWorkers.get(ThreadLocalRandom.current().nextInt(targetWorkers.size()));
}
}

51
powerjob-server/powerjob-server-remote/src/main/java/tech/powerjob/server/remote/worker/utils/SpecifyUtils.java Normal file
View File

@ -0,0 +1,51 @@
package tech.powerjob.server.remote.worker.utils;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import tech.powerjob.server.common.SJ;
import tech.powerjob.server.common.module.WorkerInfo;
import java.util.Optional;
import java.util.Set;
/**
* 指定工具
*
* @author tjq
* @since 2024/2/24
*/
public class SpecifyUtils {
private static final String TAG_EQUALS = "tagEquals:";
private static final String TAG_IN = "tagIn:";
public static boolean match(WorkerInfo workerInfo, String specifyInfo) {
String workerTag = workerInfo.getTag();
// tagIn 语法，worker 可上报多个tag，如 WorkerInfo#tag=tag1,tag2,tag3，配置中指定 tagIn=tag1 即可命中
if (specifyInfo.startsWith(TAG_IN)) {
String targetTag = specifyInfo.replace(TAG_IN, StringUtils.EMPTY);
return Optional.ofNullable(workerTag).orElse(StringUtils.EMPTY).contains(targetTag);
}
// tagEquals 语法，字符串完全匹配，worker 只可上报一个 tag，如 WorkerInfo#tag=tag1，配置中指定 tagEquals=tag1 即可命中
if (specifyInfo.startsWith(TAG_EQUALS)) {
String targetTag = specifyInfo.replace(TAG_EQUALS, StringUtils.EMPTY);
return Optional.ofNullable(workerTag).orElse(StringUtils.EMPTY).equals(targetTag);
}
// 默认情况，IP 和 tag 逗号分割后任意完全匹配即视为命中（兼容 4.3.8 版本前序逻辑）
Set<String> designatedWorkersSet = Sets.newHashSet(SJ.COMMA_SPLITTER.splitToList(specifyInfo));
for (String tagOrAddress : designatedWorkersSet) {
if (tagOrAddress.equals(workerInfo.getTag()) || tagOrAddress.equals(workerInfo.getAddress())) {
return true;
}
}
return false;
}
}

45
powerjob-server/powerjob-server-remote/src/test/java/tech/powerjob/server/remote/worker/utils/SpecifyUtilsTest.java Normal file
View File

@ -0,0 +1,45 @@
package tech.powerjob.server.remote.worker.utils;
import org.junit.jupiter.api.Test;
import tech.powerjob.server.common.module.WorkerInfo;
import static org.junit.jupiter.api.Assertions.*;
/**
* SpecifyUtilsTest
*
* @author tjq
* @since 2024/2/24
*/
class SpecifyUtilsTest {
@Test
void match() {
WorkerInfo workerInfo = new WorkerInfo();
workerInfo.setAddress("192.168.1.1");
workerInfo.setTag("tag1");
assert SpecifyUtils.match(workerInfo, "192.168.1.1");
assert SpecifyUtils.match(workerInfo, "192.168.1.1,192.168.1.2,192.168.1.3,192.168.1.4");
assert !SpecifyUtils.match(workerInfo, "172.168.1.1");
assert !SpecifyUtils.match(workerInfo, "172.168.1.1,172.168.1.2,172.168.1.3");
assert SpecifyUtils.match(workerInfo, "tag1");
assert SpecifyUtils.match(workerInfo, "tag1,tag2");
assert !SpecifyUtils.match(workerInfo, "t1");
assert !SpecifyUtils.match(workerInfo, "t1,t2");
assert SpecifyUtils.match(workerInfo, "tagIn:tag1");
assert !SpecifyUtils.match(workerInfo, "tagIn:tag2");
assert SpecifyUtils.match(workerInfo, "tagEquals:tag1");
assert !SpecifyUtils.match(workerInfo, "tagEquals:tag2");
workerInfo.setTag("tag1,tag2,tag3");
assert SpecifyUtils.match(workerInfo, "tagIn:tag1");
assert SpecifyUtils.match(workerInfo, "tagIn:tag3");
assert !SpecifyUtils.match(workerInfo, "tagIn:tag99");
}
}