geg-gas/powerjob-kingbase
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

---初始化项目

Browse Source
This commit is contained in:
张福财
2025-09-19 16:14:08 +08:00
parent 902d3d7e3b
commit afee7c03ac
767 changed files with 75809 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
powerjob-official-processors/pom.xml Normal file
View File

@ -0,0 +1,175 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>powerjob</artifactId>
<groupId>tech.powerjob</groupId>
<version>5.1.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>powerjob-official-processors</artifactId>
<name>powerjob-official-processors</name>
<version>5.1.2</version>
<packaging>jar</packaging>
<properties>
<mvn.shade.plugin.version>3.2.4</mvn.shade.plugin.version>
<!-- 不会被打包的部分，scope 只能是 test 或 provide -->
<junit.version>5.9.1</junit.version>
<logback.version>1.2.13</logback.version>
<powerjob.worker.version>5.1.2</powerjob.worker.version>
<h2.db.version>2.2.224</h2.db.version>
<mysql.version>8.0.28</mysql.version>
<spring.version>5.3.31</spring.version>
<!-- 全部 shade 化，避免依赖冲突 -->
<fastjson.version>1.2.83</fastjson.version>
<okhttp.version>3.14.9</okhttp.version>
<guava.version>30.1.1-jre</guava.version>
<commons.io.version>2.11.0</commons.io.version>
<commons.lang.version>3.10</commons.lang.version>
</properties>
<dependencies>
<!-- fastJson(used for serialization of DAG) -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>${fastjson.version}</version>
</dependency>
<!-- OKHttp -->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
</dependency>
<!-- commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>${commons.io.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons.lang.version}</version>
</dependency>
<!-- guava -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
</dependency>
<dependency>
<groupId>tech.powerjob</groupId>
<artifactId>powerjob-worker</artifactId>
<version>${powerjob.worker.version}</version>
<scope>provided</scope>
</dependency>
<!-- Spring 依赖（非强依赖） -->
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>${spring.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
<version>${spring.version}</version>
<scope>test</scope>
</dependency>
<!-- Junit tests -->
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<!-- log for test stage -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
<scope>test</scope>
</dependency>
<!-- h2 database -->
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
<version>${h2.db.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>${mysql.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>${mvn.shade.plugin.version}</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<minimizeJar>true</minimizeJar>
<relocations>
<relocation>
<pattern>okhttp3</pattern>
<shadedPattern>shade.powerjob.okhttp3</shadedPattern>
</relocation>
<relocation>
<pattern>okio</pattern>
<shadedPattern>shade.powerjob.okio</shadedPattern>
</relocation>
<relocation>
<pattern>org</pattern>
<shadedPattern>shade.powerjob.org</shadedPattern>
<excludes>
<exclude>org.slf4j.*</exclude>
<exclude>org.springframework.**</exclude>
<exclude>org.w3c.**</exclude>
</excludes>
</relocation>
<relocation>
<pattern>com.google</pattern>
<shadedPattern>shade.powerjob.com.google</shadedPattern>
</relocation>
<relocation>
<pattern>com.alibaba</pattern>
<shadedPattern>shade.powerjob.com.alibaba</shadedPattern>
</relocation>
</relocations>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

58
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/CommonBasicProcessor.java Normal file
View File

@ -0,0 +1,58 @@
package tech.powerjob.official.processors;
import tech.powerjob.official.processors.util.SecurityUtils;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.core.processor.sdk.BasicProcessor;
import tech.powerjob.worker.log.OmsLogger;
import com.google.common.base.Stopwatch;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import tech.powerjob.official.processors.util.CommonUtils;
/**
* CommonBasicProcessor
*
* @author tjq
* @since 2021/1/30
*/
@Slf4j
public abstract class CommonBasicProcessor implements BasicProcessor {
@Override
public ProcessResult process(TaskContext ctx) throws Exception {
OmsLogger omsLogger = ctx.getOmsLogger();
String securityDKey = getSecurityDKey();
if (SecurityUtils.disable(securityDKey)) {
String msg = String.format("%s is not enabled, please set '-D%s=true' to enable it", this.getClass().getSimpleName(), securityDKey);
omsLogger.warn(msg);
return new ProcessResult(false, msg);
}
String status = "unknown";
Stopwatch sw = Stopwatch.createStarted();
omsLogger.info("using params: {}", CommonUtils.parseParams(ctx));
try {
ProcessResult result = process0(ctx);
omsLogger.info("execute succeed, using {}, result: {}", sw, result);
status = result.isSuccess() ? "succeed" : "failed";
return result;
} catch (Throwable t) {
status = "exception";
omsLogger.error("execute failed!", t);
return new ProcessResult(false, ExceptionUtils.getMessage(t));
} finally {
log.info("{}|{}|{}|{}|{}", getClass().getSimpleName(), ctx.getJobId(), ctx.getInstanceId(), status, sw);
}
}
protected abstract ProcessResult process0(TaskContext taskContext) throws Exception;
protected String getSecurityDKey() {
return null;
}
}

84
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/ConfigProcessor.java Normal file
View File

@ -0,0 +1,84 @@
package tech.powerjob.official.processors.impl;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Maps;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import tech.powerjob.common.serialize.JsonUtils;
import tech.powerjob.official.processors.util.CommonUtils;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.core.processor.sdk.BroadcastProcessor;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.Optional;
/**
* 配置处理器
* 超简易的配置中心，用于配置的下发，需要配合秒级 + 广播任务使用！
* 超低成本下的解决方案，强配置 or 高SLA 场景，请使用标准的配置管理中间件。
* 外部调用方法 {@link ConfigProcessor#fetchConfig()}
*
* @author tjq
* @since 2022/9/17
*/
@Slf4j
public class ConfigProcessor implements BroadcastProcessor {
/**
* 获取配置
* @return 控制台下发的配置
*/
public static Map<String, Object> fetchConfig() {
if (config == null) {
return Maps.newHashMap();
}
return Optional.ofNullable(config.getConfig()).orElse(Maps.newHashMap());
}
private static Config config;
@Override
public ProcessResult process(TaskContext context) throws Exception {
Config newCfg = JsonUtils.parseObject(CommonUtils.parseParams(context), Config.class);
context.getOmsLogger().info("[ConfigProcessor] receive and update config: {}", config);
// 空场景不更新
final Map<String, Object> realConfig = newCfg.config;
if (realConfig == null) {
return new ProcessResult(false, "CONFIG_IS_NULL");
}
config = newCfg;
if (StringUtils.isNotEmpty(config.persistentFileName)) {
final File file = new File(config.persistentFileName);
String content = JSONObject.toJSONString(realConfig);
FileUtils.copyToFile(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)), file);
}
return new ProcessResult(true, "UPDATE_SUCCESS");
}
@Data
public static class Config implements Serializable {
/**
* 原始配置
*/
private Map<String, Object> config;
/**
* 持久到本地的全路径名称
*/
private String persistentFileName;
}
}

111
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/FileCleanupProcessor.java Normal file
View File

@ -0,0 +1,111 @@
package tech.powerjob.official.processors.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import tech.powerjob.official.processors.util.SecurityUtils;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.core.processor.sdk.BroadcastProcessor;
import tech.powerjob.worker.log.OmsLogger;
import com.google.common.base.Stopwatch;
import lombok.Data;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import java.io.File;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.atomic.LongAdder;
import java.util.regex.Pattern;
/**
* common file(like logs) cleaner
*
* @author tjq
* @since 2021/2/1
*/
public class FileCleanupProcessor implements BroadcastProcessor {
@Override
public ProcessResult preProcess(TaskContext context) throws Exception {
if (SecurityUtils.disable(SecurityUtils.ENABLE_FILE_CLEANUP_PROCESSOR)) {
String msg = String.format("FileCleanupProcessor is not enabled, please set '-D%s=true' to enable it", SecurityUtils.ENABLE_FILE_CLEANUP_PROCESSOR);
context.getOmsLogger().warn(msg);
return new ProcessResult(false, msg);
}
return new ProcessResult(true);
}
@Override
public ProcessResult process(TaskContext taskContext) throws Exception {
OmsLogger logger = taskContext.getOmsLogger();
logger.info("using params: {}", taskContext.getJobParams());
LongAdder cleanNum = new LongAdder();
Stopwatch sw = Stopwatch.createStarted();
List<CleanupParams> cleanupParamsList = JSONArray.parseArray(taskContext.getJobParams(), CleanupParams.class);
cleanupParamsList.forEach(params -> {
logger.info("start to process: {}", JSON.toJSON(params));
if (StringUtils.isEmpty(params.filePattern) || StringUtils.isEmpty(params.dirPath)) {
logger.warn("skip due to invalid params!");
return;
}
File dir = new File(params.dirPath);
if (!dir.exists()) {
logger.warn("skip due to dirPath[{}] not exists", params.dirPath);
return;
}
if (!dir.isDirectory()) {
logger.warn("skip due to dirPath[{}] is not a directory", params.dirPath);
return;
}
logger.info("start to search directory: {}", params.dirPath);
Collection<File> files = FileUtils.listFiles(dir, null, true);
logger.info("total file num: {}", files.size());
Pattern filePattern = Pattern.compile(params.filePattern);
files.forEach(file -> {
String fileName = file.getName();
String filePath = file.getAbsolutePath();
if (!filePattern.matcher(fileName).matches()) {
logger.info("file[{}] won't be deleted due to filename not match the pattern: {}", fileName, params.filePattern);
return;
}
// last modify time interval, xxx hours
int interval = (int) Math.ceil((System.currentTimeMillis() - file.lastModified()) / 3600000.0);
if (interval < params.retentionTime) {
logger.info("file[{}] won't be deleted because it does not meet the time requirement", filePath);
return;
}
try {
FileUtils.forceDelete(file);
cleanNum.increment();
logger.info("delete file[{}] successfully!", filePath);
} catch (Exception e) {
logger.error("delete file[{}] failed!", filePath, e);
}
});
});
return new ProcessResult(true, String.format("cost:%s,clean:%d", sw.toString(), cleanNum.longValue()));
}
@Data
public static class CleanupParams {
private String dirPath;
private String filePattern;
private Integer retentionTime;
}
}

162
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/HttpProcessor.java Normal file
View File

@ -0,0 +1,162 @@
package tech.powerjob.official.processors.impl;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.JSONValidator;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.log.OmsLogger;
import lombok.Data;
import okhttp3.*;
import org.apache.commons.lang3.StringUtils;
import tech.powerjob.official.processors.CommonBasicProcessor;
import tech.powerjob.official.processors.util.CommonUtils;
import java.time.Duration;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
/**
* common http processor
*
* @author tjq
* @author Jiang Jining
* @since 2021/1/30
*/
public class HttpProcessor extends CommonBasicProcessor {
/**
* Default timeout is 60 seconds.
*/
private static final int DEFAULT_TIMEOUT = 60;
private static final int HTTP_SUCCESS_CODE = 200;
private static final Map<Integer, OkHttpClient> CLIENT_STORE = new ConcurrentHashMap<>();
@Override
public ProcessResult process0(TaskContext taskContext) throws Exception {
OmsLogger omsLogger = taskContext.getOmsLogger();
HttpParams httpParams = JSON.parseObject(CommonUtils.parseParams(taskContext), HttpParams.class);
if (httpParams == null) {
String message = "httpParams is null, please check jobParam configuration.";
omsLogger.warn(message);
return new ProcessResult(false, message);
}
if (StringUtils.isEmpty(httpParams.url)) {
return new ProcessResult(false, "url can't be empty!");
}
if (!httpParams.url.startsWith("http")) {
httpParams.url = "http://" + httpParams.url;
}
omsLogger.info("request url: {}", httpParams.url);
// set default method
if (StringUtils.isEmpty(httpParams.method)) {
httpParams.method = "GET";
omsLogger.info("using default request method: GET");
} else {
httpParams.method = httpParams.method.toUpperCase();
omsLogger.info("request method: {}", httpParams.method);
}
// set default mediaType
if (!"GET".equals(httpParams.method)) {
// set default request body
if (StringUtils.isEmpty(httpParams.body)) {
httpParams.body = new JSONObject().toJSONString();
omsLogger.warn("try to use default request body:{}", httpParams.body);
}
if (JSONValidator.from(httpParams.body).validate() && StringUtils.isEmpty(httpParams.mediaType)) {
httpParams.mediaType = "application/json";
omsLogger.warn("try to use 'application/json' as media type");
}
}
// set default timeout
if (httpParams.timeout == null) {
httpParams.timeout = DEFAULT_TIMEOUT;
}
omsLogger.info("request timeout: {} seconds", httpParams.timeout);
OkHttpClient client = getClient(httpParams.timeout);
Request.Builder builder = new Request.Builder().url(httpParams.url);
if (httpParams.headers != null) {
httpParams.headers.forEach((k, v) -> {
builder.addHeader(k, v);
omsLogger.info("add header {}:{}", k, v);
});
}
switch (httpParams.method) {
case "PUT":
case "DELETE":
case "POST":
MediaType mediaType = MediaType.parse(httpParams.mediaType);
omsLogger.info("mediaType: {}", mediaType);
RequestBody requestBody = RequestBody.create(mediaType, httpParams.body);
builder.method(httpParams.method, requestBody);
break;
default:
builder.get();
}
Response response = client.newCall(builder.build()).execute();
omsLogger.info("response: {}", response);
String msgBody = "";
if (response.body() != null) {
msgBody = response.body().string();
}
int responseCode = response.code();
String res = String.format("code:%d, body:%s", responseCode, msgBody);
boolean success = true;
if (responseCode != HTTP_SUCCESS_CODE) {
success = false;
omsLogger.warn("{} url: {} failed, response code is {}, response body is {}",
httpParams.method, httpParams.url, responseCode, msgBody);
}
return new ProcessResult(success, res);
}
@Data
public static class HttpParams {
/**
* POST / GET / PUT / DELETE
*/
private String method;
/**
* the request url
*/
private String url;
/**
* application/json
* application/xml
* image/png
* image/jpeg
* image/gif
*/
private String mediaType;
private String body;
private Map<String, String> headers;
/**
* timeout for complete calls
*/
private Integer timeout;
}
private static OkHttpClient getClient(Integer timeout) {
return CLIENT_STORE.computeIfAbsent(timeout, ignore -> new OkHttpClient.Builder()
.connectTimeout(Duration.ZERO)
.readTimeout(Duration.ZERO)
.writeTimeout(Duration.ZERO)
.callTimeout(timeout, TimeUnit.SECONDS)
.build());
}
}

221
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/VerificationProcessor.java Normal file
View File

@ -0,0 +1,221 @@
package tech.powerjob.official.processors.impl;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import lombok.*;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import tech.powerjob.common.exception.PowerJobException;
import tech.powerjob.common.utils.NetUtils;
import tech.powerjob.official.processors.CommonBasicProcessor;
import tech.powerjob.official.processors.util.CommonUtils;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.core.processor.TaskResult;
import tech.powerjob.worker.core.processor.sdk.BroadcastProcessor;
import tech.powerjob.worker.core.processor.sdk.MapReduceProcessor;
import tech.powerjob.worker.log.OmsLogger;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ThreadLocalRandom;
/**
* 功能验证用处理器，帮助用户快速验证想要测试的功能
*
* @author tjq
* @since 2023/8/13
*/
public class VerificationProcessor extends CommonBasicProcessor implements MapReduceProcessor, BroadcastProcessor {
@Override
protected ProcessResult process0(TaskContext taskContext) throws Exception {
final OmsLogger omsLogger = taskContext.getOmsLogger();
final String paramsStr = CommonUtils.parseParams(taskContext);
final VerificationParam verificationParam = StringUtils.isEmpty(paramsStr) ? new VerificationParam() : JSONObject.parseObject(paramsStr, VerificationParam.class);
final Mode mode = Mode.of(verificationParam.getMode());
omsLogger.info("META: {}", taskContext.getInstanceMeta());
switch (mode) {
case ERROR:
return new ProcessResult(false, "EXECUTE_FAILED_FOR_TEST");
case EXCEPTION:
throw new PowerJobException("exception for test");
case TIMEOUT:
final Long sleepMs = Optional.ofNullable(verificationParam.getSleepMs()).orElse(3600000L);
Thread.sleep(sleepMs);
return new ProcessResult(true, "AFTER_SLEEP_" + sleepMs);
case RETRY:
int currentRetryTimes = taskContext.getCurrentRetryTimes();
int maxRetryTimes = taskContext.getMaxRetryTimes();
omsLogger.info("[Retry] currentRetryTimes: {}, maxRetryTimes: {}", currentRetryTimes, maxRetryTimes);
if (currentRetryTimes < maxRetryTimes) {
Thread.sleep(100);
omsLogger.info("[Retry] currentRetryTimes[{}] < maxRetryTimes[{}], return failed status!", currentRetryTimes, maxRetryTimes);
return new ProcessResult(false, "FAILED_UNTIL_LAST_RETRY_" + currentRetryTimes);
} else {
omsLogger.info("[Retry] last retry, return success status!");
return new ProcessResult(true, "RETRY_SUCCESSFULLY!");
}
case MR:
if (isRootTask()) {
final int batchNum = Optional.ofNullable(verificationParam.getBatchNum()).orElse(10);
final int batchSize = Optional.ofNullable(verificationParam.getBatchSize()).orElse(100);
omsLogger.info("[VerificationProcessor] start root task~");
List<TestSubTask> subTasks = new ArrayList<>();
for (int a = 0; a < batchNum; a++) {
for (int b = 0; b < batchSize; b++) {
int x = a * batchSize + b;
subTasks.add(new TestSubTask("task_" + x, x));
}
map(subTasks, "MAP_TEST_TASK_" + a);
omsLogger.info("[VerificationProcessor] [{}] map one batch successfully~", batchNum);
subTasks.clear();
}
omsLogger.info("[VerificationProcessor] all map successfully!");
return new ProcessResult(true, "MAP_SUCCESS");
} else {
String taskId = taskContext.getTaskId();
final Double successRate = Optional.ofNullable(verificationParam.getSubTaskSuccessRate()).orElse(0.5);
final double rd = ThreadLocalRandom.current().nextDouble(0, 1);
boolean success = rd <= successRate;
long processCost = ThreadLocalRandom.current().nextLong(277);
Thread.sleep(processCost);
omsLogger.info("[VerificationProcessor] [MR] taskId:{}, processCost: {}, success:{}", taskId, processCost, success);
return new ProcessResult(success, RandomStringUtils.randomAlphanumeric(3));
}
}
String randomMsg = RandomStringUtils.randomAlphanumeric(Optional.ofNullable(verificationParam.getResponseSize()).orElse(10));
omsLogger.info("generate random string: {}", randomMsg);
return new ProcessResult(true, "EXECUTE_SUCCESSFULLY_" + randomMsg);
}
@Override
public ProcessResult reduce(TaskContext context, List<TaskResult> taskResults) {
List<String> successTaskIds = Lists.newArrayList();
List<String> failedTaskIds = Lists.newArrayList();
StringBuilder sb = new StringBuilder();
taskResults.forEach(taskResult -> {
sb.append("tId:").append(taskResult.getTaskId()).append(";")
.append("tSuc:").append(taskResult.isSuccess()).append(";")
.append("tRes:").append(taskResult.getResult());
if (taskResult.isSuccess()) {
successTaskIds.add(taskResult.getTaskId());
} else {
failedTaskIds.add(taskResult.getTaskId());
}
});
context.getOmsLogger().info("[Reduce] [summary] successTaskNum: {}, failedTaskNum: {}, successRate: {}",
successTaskIds.size(), failedTaskIds.size(), 1.0 * successTaskIds.size() / (successTaskIds.size() + failedTaskIds.size()));
context.getOmsLogger().info("[Reduce] successTaskIds: {}", successTaskIds);
context.getOmsLogger().info("[Reduce] failedTaskIds: {}", failedTaskIds);
return new ProcessResult(true, sb.toString());
}
/* ************************** 广播任务部分 ************************** */
@Override
public ProcessResult preProcess(TaskContext context) throws Exception {
context.getOmsLogger().info("start to preProcess, current worker IP is {}.", NetUtils.getLocalHost4Test());
return new ProcessResult(true, "preProcess successfully!");
}
@Override
public ProcessResult postProcess(TaskContext context, List<TaskResult> taskResults) throws Exception {
OmsLogger omsLogger = context.getOmsLogger();
omsLogger.info("start to postProcess, current worker IP is {}.", NetUtils.getLocalHost4Test());
omsLogger.info("====== All Node's Process Result ======");
taskResults.forEach(r -> omsLogger.info("taskId:{},success:{},result:{}", r.getTaskId(), r.isSuccess(), r.getResult()));
return new ProcessResult(true, "postProcess successfully!");
}
/* ************************** 广播任务部分 ************************** */
enum Mode {
/**
* 常规模式，直接返回响应
* {"mode":"BASE","responseSize":12}
*/
BASE,
/**
* 超时，sleep 一段时间测试超时控制
* {"mode":"TIMEOUT","sleepMs":3600000}
*/
TIMEOUT,
/**
* 测试执行失败，响应返回 success = false
* {"mode":"ERROR"}
*/
ERROR,
/**
* 测试执行异常，抛出异常
* {"mode":"EXCEPTION"}
*/
EXCEPTION,
/**
* MapReduce，需要控制台配置为 MapReduce 执行模式
* {"mode":"MR","batchNum": 10, "batchSize": 20,"subTaskSuccessRate":0.7}
*/
MR,
/**
* 重试后成功，JOB 配置 Task 重试次数
* {"mode":"EXCEPTION"}
*/
RETRY
;
public static Mode of(String v) {
for (Mode m : values()) {
if (m.name().equalsIgnoreCase(v)) {
return m;
}
}
return Mode.BASE;
}
}
@Data
public static class VerificationParam implements Serializable {
/**
* 验证模式
*/
private String mode;
/**
* 休眠时间，用于验证超时
*/
private Long sleepMs;
/**
* 【MR】批次大小，用于验证 MapReduce
*/
private Integer batchSize;
/**
* 【MR】batchNum
*/
private Integer batchNum;
/**
* 【MR】子任务成功率
*/
private Double subTaskSuccessRate;
private Integer responseSize;
}
@Getter
@ToString
@NoArgsConstructor
@AllArgsConstructor
public static class TestSubTask {
private String taskName;
private int id;
}
}

40
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/context/InjectWorkflowContextProcessor.java Normal file
View File

@ -0,0 +1,40 @@
package tech.powerjob.official.processors.impl.context;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.TypeReference;
import tech.powerjob.official.processors.CommonBasicProcessor;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.core.processor.WorkflowContext;
import tech.powerjob.worker.log.OmsLogger;
import java.util.HashMap;
import java.util.Map;
/**
* @author Echo009
* @since 2022/2/16
*/
public class InjectWorkflowContextProcessor extends CommonBasicProcessor {
@Override
protected ProcessResult process0(TaskContext taskContext) {
String jobParams = taskContext.getJobParams();
OmsLogger omsLogger = taskContext.getOmsLogger();
try {
HashMap<String, Object> data = JSON.parseObject(jobParams, new TypeReference<HashMap<String, Object>>() {
});
WorkflowContext workflowContext = taskContext.getWorkflowContext();
for (Map.Entry<String, Object> entry : data.entrySet()) {
workflowContext.appendData2WfContext(entry.getKey(), entry.getValue());
omsLogger.info("inject context, {}:{}", entry.getKey(), entry.getValue());
}
} catch (Exception e) {
omsLogger.error("Fail to parse job params:{},it is not a valid json string!", jobParams, e);
return new ProcessResult(false);
}
return new ProcessResult(true);
}
}

186
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/script/AbstractScriptProcessor.java Normal file
View File

@ -0,0 +1,186 @@
package tech.powerjob.official.processors.impl.script;
import tech.powerjob.common.utils.SysUtils;
import tech.powerjob.worker.common.utils.PowerFileUtils;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.log.OmsLogger;
import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import tech.powerjob.official.processors.CommonBasicProcessor;
import tech.powerjob.official.processors.util.CommonUtils;
import java.io.*;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.util.Set;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinTask;
/**
* 脚本处理器
*
* @author tjq
* @author Jiang Jining
* @since 2020/4/16
*/
@Slf4j
public abstract class AbstractScriptProcessor extends CommonBasicProcessor {
private static final ForkJoinPool POOL = new ForkJoinPool(4 * SysUtils.availableProcessors());
private static final Set<String> DOWNLOAD_PROTOCOL = Sets.newHashSet("http", "https", "ftp");
protected static final String SH_SHELL = "/bin/sh";
protected static final String CMD_SHELL = "cmd.exe";
private static final String WORKER_DIR = PowerFileUtils.workspace() + "/official_script_processor/";
@Override
protected ProcessResult process0(TaskContext context) throws Exception {
OmsLogger omsLogger = context.getOmsLogger();
String scriptParams = CommonUtils.parseParams(context);
omsLogger.info("[SYSTEM] ScriptProcessor start to process, params: {}", scriptParams);
if (scriptParams == null) {
String message = "[SYSTEM] ScriptParams is null, please check jobParam configuration.";
omsLogger.warn(message);
return new ProcessResult(false, message);
}
String scriptPath = prepareScriptFile(context.getInstanceId(), scriptParams);
omsLogger.info("[SYSTEM] Generate executable file successfully, path: {}", scriptPath);
if (SystemUtils.IS_OS_WINDOWS) {
if (StringUtils.equals(getRunCommand(), SH_SHELL)) {
String message = String.format("[SYSTEM] Current OS is %s where shell scripts cannot run.", SystemUtils.OS_NAME);
omsLogger.warn(message);
return new ProcessResult(false, message);
}
}
// 授权
if ( !SystemUtils.IS_OS_WINDOWS) {
ProcessBuilder chmodPb = new ProcessBuilder("/bin/chmod", "755", scriptPath);
// 等待返回，这里不可能导致死锁（shell产生大量数据可能导致死锁）
chmodPb.start().waitFor();
omsLogger.info("[SYSTEM] chmod 755 authorization complete, ready to start execution~");
}
// 2. 执行目标脚本
ProcessBuilder pb = StringUtils.equals(getRunCommand(), CMD_SHELL) ?
new ProcessBuilder(getRunCommand(), "/c", scriptPath)
: new ProcessBuilder(getRunCommand(), scriptPath);
Process process = pb.start();
StringBuilder inputBuilder = new StringBuilder();
StringBuilder errorBuilder = new StringBuilder();
boolean success = true;
String result;
final Charset charset = getCharset();
try {
InputStream is = process.getInputStream();
InputStream es = process.getErrorStream();
ForkJoinTask<?> inputSubmit = POOL.submit(() -> copyStream(is, inputBuilder, omsLogger, charset));
ForkJoinTask<?> errorSubmit = POOL.submit(() -> copyStream(es, errorBuilder, omsLogger, charset));
success = process.waitFor() == 0;
// 阻塞等待日志读取
inputSubmit.get();
errorSubmit.get();
} catch (InterruptedException ie) {
omsLogger.info("[SYSTEM] ScriptProcessor has been interrupted");
} finally {
result = String.format("[INPUT]: %s;[ERROR]: %s", inputBuilder, errorBuilder);
}
return new ProcessResult(success, result);
}
private String prepareScriptFile(Long instanceId, String processorInfo) throws IOException {
String scriptPath = WORKER_DIR + getScriptName(instanceId);
File script = new File(scriptPath);
if (script.exists()) {
return scriptPath;
}
File dir = new File(script.getParent());
boolean success = dir.mkdirs();
success = script.createNewFile();
if (!success) {
throw new RuntimeException("create script file failed");
}
// 如果是下载链接，则从网络获取
for (String protocol : DOWNLOAD_PROTOCOL) {
if (processorInfo.startsWith(protocol)) {
FileUtils.copyURLToFile(new URL(processorInfo), script, 5000, 300000);
return scriptPath;
}
}
final Charset charset = getCharset();
if(charset != null)
{
try (Writer fstream = new OutputStreamWriter(Files.newOutputStream(script.toPath()), charset); BufferedWriter out = new BufferedWriter(fstream)) {
out.write(processorInfo);
out.flush();
}
}
else {
try (FileWriter fw = new FileWriter(script); BufferedWriter bw = new BufferedWriter(fw)) {
bw.write(processorInfo);
bw.flush();
}
}
return scriptPath;
}
private void copyStream(InputStream is, StringBuilder sb, OmsLogger omsLogger, Charset charset) {
String line;
try (BufferedReader br = new BufferedReader(new InputStreamReader(is, charset))) {
while ((line = br.readLine()) != null) {
sb.append(line).append(System.lineSeparator());
// 同步到在线日志
omsLogger.info(line);
}
} catch (Exception e) {
log.warn("[ScriptProcessor] copyStream failed.", e);
omsLogger.warn("[SYSTEM] copyStream failed.", e);
sb.append("Exception: ").append(e);
} finally {
try {
is.close();
} catch (IOException e) {
log.warn("[ScriptProcessor] close stream failed.", e);
omsLogger.warn("[SYSTEM] close stream failed.", e);
}
}
}
/**
* 生成脚本名称
* @param instanceId id of instance
* @return 文件名称
*/
protected abstract String getScriptName(Long instanceId);
/**
* 获取运行命令（eg，shell返回 /bin/sh）
* @return 执行脚本的命令
*/
protected abstract String getRunCommand();
/**
* 默认不指定
* @return Charset
*/
protected Charset getCharset() {
return StandardCharsets.UTF_8;
}
}

27
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/script/CMDProcessor.java Normal file
View File

@ -0,0 +1,27 @@
package tech.powerjob.official.processors.impl.script;
import java.nio.charset.Charset;
/**
* CMDProcessor
*
* @author fddc
* @since 2021/5/14
*/
public class CMDProcessor extends AbstractScriptProcessor {
@Override
protected String getScriptName(Long instanceId) {
return String.format("cmd_%d.bat", instanceId);
}
@Override
protected String getRunCommand() {
return "cmd.exe";
}
@Override
protected Charset getCharset() {
return Charset.defaultCharset();
}
}

27
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/script/PowerShellProcessor.java Normal file
View File

@ -0,0 +1,27 @@
package tech.powerjob.official.processors.impl.script;
import java.nio.charset.Charset;
/**
* PowerShellProcessor
*
* @author fddc
* @since 2021/5/14
*/
public class PowerShellProcessor extends AbstractScriptProcessor {
@Override
protected String getScriptName(Long instanceId) {
return String.format("powershell_%d.ps1", instanceId);
}
@Override
protected String getRunCommand() {
return "powershell.exe";
}
@Override
protected Charset getCharset() {
return Charset.defaultCharset();
}
}

20
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/script/PythonProcessor.java Normal file
View File

@ -0,0 +1,20 @@
package tech.powerjob.official.processors.impl.script;
/**
* python processor
*
* @author tjq
* @since 2021/2/7
*/
public class PythonProcessor extends AbstractScriptProcessor {
@Override
protected String getScriptName(Long instanceId) {
return String.format("python_%d.py", instanceId);
}
@Override
protected String getRunCommand() {
return "python";
}
}

20
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/script/ShellProcessor.java Normal file
View File

@ -0,0 +1,20 @@
package tech.powerjob.official.processors.impl.script;
/**
* shell processor
*
* @author tjq
* @since 2021/2/7
*/
public class ShellProcessor extends AbstractScriptProcessor {
@Override
protected String getScriptName(Long instanceId) {
return String.format("shell_%d.sh", instanceId);
}
@Override
protected String getRunCommand() {
return SH_SHELL;
}
}

267
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/sql/AbstractSqlProcessor.java Normal file
View File

@ -0,0 +1,267 @@
package tech.powerjob.official.processors.impl.sql;
import com.alibaba.fastjson.JSON;
import com.google.common.base.Joiner;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StopWatch;
import tech.powerjob.official.processors.CommonBasicProcessor;
import tech.powerjob.official.processors.util.CommonUtils;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.log.OmsLogger;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
/**
* SQL Processor
*
* 处理流程：
* * * 解析参数 => 校验参数 => 解析 SQL => 校验 SQL => 执行 SQL
*
* 可以通过 {@link AbstractSqlProcessor#registerSqlValidator} 方法注册 SQL 校验器拦截非法 SQL
* 可以通过指定 {@link AbstractSqlProcessor.SqlParser} 来实现定制 SQL 解析逻辑的需求（比如 宏变量替换，参数替换等）
*
* @author Echo009
* @since 2021/3/12
*/
@Slf4j
public abstract class AbstractSqlProcessor extends CommonBasicProcessor {
/**
* 默认超时时间
*/
protected static final int DEFAULT_TIMEOUT = 60;
/**
* name => SQL validator
* 注意 ：
* - 返回 true 表示验证通过
* - 返回 false 表示 SQL 非法，将被拒绝执行
*/
protected final Map<String, Predicate<String>> sqlValidatorMap = Maps.newConcurrentMap();
/**
* 自定义 SQL 解析器
*/
protected SqlParser sqlParser;
private static final Joiner JOINER = Joiner.on("|").useForNull("-");
@Override
public ProcessResult process0(TaskContext taskContext) {
OmsLogger omsLogger = taskContext.getOmsLogger();
// 解析参数
SqlParams sqlParams = extractParams(taskContext);
omsLogger.info("origin sql params: {}", JSON.toJSON(sqlParams));
// 校验参数
validateParams(sqlParams);
StopWatch stopWatch = new StopWatch(this.getClass().getSimpleName());
// 解析
stopWatch.start("Parse SQL");
if (sqlParser != null) {
omsLogger.info("before parse sql: {}", sqlParams.getSql());
String newSQL = sqlParser.parse(sqlParams.getSql(), taskContext);
sqlParams.setSql(newSQL);
omsLogger.info("after parse sql: {}", newSQL);
}
stopWatch.stop();
// 校验 SQL
stopWatch.start("Validate SQL");
validateSql(sqlParams.getSql(), omsLogger);
stopWatch.stop();
// 执行
stopWatch.start("Execute SQL");
omsLogger.info("final sql params: {}", JSON.toJSON(sqlParams));
executeSql(sqlParams, taskContext);
stopWatch.stop();
omsLogger.info(stopWatch.prettyPrint());
String message = String.format("execute successfully, used time: %s millisecond", stopWatch.getTotalTimeMillis());
return new ProcessResult(true, message);
}
abstract Connection getConnection(SqlParams sqlParams, TaskContext taskContext) throws SQLException;
/**
* 执行 SQL
* @param sqlParams SQL processor 参数信息
* @param ctx 任务上下文
*/
@SneakyThrows
private void executeSql(SqlParams sqlParams, TaskContext ctx) {
OmsLogger omsLogger = ctx.getOmsLogger();
boolean originAutoCommitFlag ;
try (Connection connection = getConnection(sqlParams, ctx)) {
originAutoCommitFlag = connection.getAutoCommit();
connection.setAutoCommit(false);
try (Statement statement = connection.createStatement()) {
statement.setQueryTimeout(sqlParams.getTimeout() == null ? DEFAULT_TIMEOUT : sqlParams.getTimeout());
statement.execute(sqlParams.getSql());
connection.commit();
if (sqlParams.showResult) {
outputSqlResult(statement, omsLogger);
}
} catch (Throwable e) {
omsLogger.error("execute sql failed, try to rollback", e);
connection.rollback();
throw e;
} finally {
connection.setAutoCommit(originAutoCommitFlag);
}
}
}
private void outputSqlResult(Statement statement, OmsLogger omsLogger) throws SQLException {
omsLogger.info("====== SQL EXECUTE RESULT ======");
for (int index = 0; index < Integer.MAX_VALUE; index++) {
// 某一个结果集
ResultSet resultSet = statement.getResultSet();
if (resultSet != null) {
try (ResultSet rs = resultSet) {
int columnCount = rs.getMetaData().getColumnCount();
List<String> columnNames = Lists.newLinkedList();
//column – the first column is 1, the second is 2, ...
for (int i = 1; i <= columnCount; i++) {
columnNames.add(rs.getMetaData().getColumnName(i));
}
omsLogger.info("[Result-{}] [Columns] {}" + System.lineSeparator(), index, JOINER.join(columnNames));
int rowIndex = 0;
List<Object> row = Lists.newLinkedList();
while (rs.next()) {
for (int i = 1; i <= columnCount; i++) {
row.add(rs.getObject(i));
}
omsLogger.info("[Result-{}] [Row-{}] {}" + System.lineSeparator(), index, rowIndex++, JOINER.join(row));
}
}
} else {
int updateCount = statement.getUpdateCount();
if (updateCount != -1) {
omsLogger.info("[Result-{}] update count: {}", index, updateCount);
}
}
if (((!statement.getMoreResults()) && (statement.getUpdateCount() == -1))) {
break;
}
}
omsLogger.info("====== SQL EXECUTE RESULT ======");
}
/**
* 提取参数信息
*
* @param taskContext 任务上下文
* @return SqlParams
*/
protected SqlParams extractParams(TaskContext taskContext) {
return JSON.parseObject(CommonUtils.parseParams(taskContext), SqlParams.class);
}
/**
* 校验参数，如果校验不通过直接抛异常
*
* @param sqlParams SQL 参数信息
*/
protected void validateParams(SqlParams sqlParams) {
// do nothing
}
/**
* 设置 SQL 验证器
*
* @param sqlParser SQL 解析器
*/
public void setSqlParser(SqlParser sqlParser) {
this.sqlParser = sqlParser;
}
/**
* 注册一个 SQL 验证器
*
* @param validatorName 验证器名称
* @param sqlValidator 验证器
*/
public void registerSqlValidator(String validatorName, Predicate<String> sqlValidator) {
sqlValidatorMap.put(validatorName, sqlValidator);
log.info("register sql validator({})' successfully.", validatorName);
}
/**
* 校验 SQL 合法性
*/
private void validateSql(String sql, OmsLogger omsLogger) {
if (sqlValidatorMap.isEmpty()) {
return;
}
for (Map.Entry<String, Predicate<String>> entry : sqlValidatorMap.entrySet()) {
Predicate<String> validator = entry.getValue();
if (!validator.test(sql)) {
omsLogger.error("validate sql by validator[{}] failed, skip to process!", entry.getKey());
throw new IllegalArgumentException("illegal sql, can't pass the validation of " + entry.getKey());
}
}
}
@Data
public static class SqlParams {
/**
* 数据源名称
*/
private String dataSourceName;
/**
* 需要执行的 SQL
*/
private String sql;
/**
* 超时时间
*/
private Integer timeout;
/**
* jdbc url
* 具体格式可参考 https://www.baeldung.com/java-jdbc-url-format
*/
private String jdbcUrl;
/**
* 是否展示 SQL 执行结果
*/
private boolean showResult;
}
@FunctionalInterface
public interface SqlParser {
/**
* 自定义 SQL 解析逻辑
*
* @param sql 原始 SQL 语句
* @param taskContext 任务上下文
* @return 解析后的 SQL
*/
String parse(String sql, TaskContext taskContext);
}
}

45
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/sql/DynamicDatasourceSqlProcessor.java Normal file
View File

@ -0,0 +1,45 @@
package tech.powerjob.official.processors.impl.sql;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import tech.powerjob.official.processors.util.CommonUtils;
import tech.powerjob.official.processors.util.SecurityUtils;
import tech.powerjob.worker.core.processor.TaskContext;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
/**
* execute sql by dynamic datasource, which is very dangerous but can save your life at the critical moment
*
* @author tjq
* @since 2021/3/14
*/
public class DynamicDatasourceSqlProcessor extends AbstractSqlProcessor {
@Override
protected void validateParams(SqlParams sqlParams) {
if (StringUtils.isEmpty(sqlParams.getJdbcUrl())) {
throw new IllegalArgumentException("jdbcUrl can't be empty in DynamicDatasourceSqlProcessor!");
}
}
@Override
Connection getConnection(SqlParams sqlParams, TaskContext taskContext) throws SQLException {
JSONObject params = JSONObject.parseObject(CommonUtils.parseParams(taskContext));
Properties properties = new Properties();
// normally at least a "user" and "password" property should be included
params.forEach((k, v) -> properties.setProperty(k, String.valueOf(v)));
return DriverManager.getConnection(sqlParams.getJdbcUrl(), properties);
}
@Override
protected String getSecurityDKey() {
return SecurityUtils.ENABLE_DYNAMIC_SQL_PROCESSOR;
}
}

95
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/impl/sql/SpringDatasourceSqlProcessor.java Normal file
View File

@ -0,0 +1,95 @@
package tech.powerjob.official.processors.impl.sql;
import tech.powerjob.worker.core.processor.TaskContext;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Map;
import java.util.Objects;
/**
* 简单 Spring SQL 处理器，目前只能用 Spring Bean 的方式加载
* 直接忽略 SQL 执行的返回值
*
* 注意 :
* 默认情况下没有过滤任何 SQL
* 建议生产环境一定要使用 {@link SpringDatasourceSqlProcessor#registerSqlValidator} 方法注册至少一个校验器拦截非法 SQL
*
* 默认情况下会直接执行参数中的 SQL
* 可以通过添加 {@link SpringDatasourceSqlProcessor.SqlParser} 来实现定制 SQL 解析逻辑的需求（比如 宏变量替换，参数替换等）
*
* @author Echo009
* @since 2021/3/10
*/
@Slf4j
public class SpringDatasourceSqlProcessor extends AbstractSqlProcessor {
/**
* 默认的数据源名称
*/
private static final String DEFAULT_DATASOURCE_NAME = "default";
/**
* name => data source
*/
private final Map<String, DataSource> dataSourceMap;
/**
* 指定默认的数据源
*
* @param defaultDataSource 默认数据源
*/
public SpringDatasourceSqlProcessor(DataSource defaultDataSource) {
dataSourceMap = Maps.newConcurrentMap();
registerDataSource(DEFAULT_DATASOURCE_NAME, defaultDataSource);
}
@Override
Connection getConnection(SqlParams sqlParams, TaskContext taskContext) throws SQLException {
return dataSourceMap.get(sqlParams.getDataSourceName()).getConnection();
}
/**
* 校验参数，如果校验不通过直接抛异常
*
* @param sqlParams SQL 参数信息
*/
@Override
protected void validateParams(SqlParams sqlParams) {
// 检查数据源
if (StringUtils.isEmpty(sqlParams.getDataSourceName())) {
// use the default data source when current data source name is empty
sqlParams.setDataSourceName(DEFAULT_DATASOURCE_NAME);
}
dataSourceMap.computeIfAbsent(sqlParams.getDataSourceName(), dataSourceName -> {
throw new IllegalArgumentException("can't find data source with name " + dataSourceName);
});
}
/**
* 注册数据源
*
* @param dataSourceName 数据源名称
* @param dataSource 数据源
*/
public void registerDataSource(String dataSourceName, DataSource dataSource) {
Objects.requireNonNull(dataSourceName, "DataSource name must not be null");
Objects.requireNonNull(dataSource, "DataSource must not be null");
dataSourceMap.put(dataSourceName, dataSource);
log.info("register data source({})' successfully.", dataSourceName);
}
/**
* 移除数据源
*
* @param dataSourceName 数据源名称
*/
public void removeDataSource(String dataSourceName) {
DataSource remove = dataSourceMap.remove(dataSourceName);
if (remove != null) {
log.warn("remove data source({})' successfully.", dataSourceName);
}
}
}

28
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/util/CommonUtils.java Normal file
View File

@ -0,0 +1,28 @@
package tech.powerjob.official.processors.util;
import tech.powerjob.worker.core.processor.TaskContext;
import org.apache.commons.lang3.StringUtils;
/**
* CommonUtils
*
* @author tjq
* @since 2021/2/1
*/
public class CommonUtils {
private CommonUtils() {
}
public static String parseParams(TaskContext context) {
// 工作流中的总是优先使用 jobParams
if (context.getWorkflowContext().getWfInstanceId() != null) {
return context.getJobParams();
}
if (StringUtils.isNotEmpty(context.getInstanceParams())) {
return context.getInstanceParams();
}
return context.getJobParams();
}
}

24
powerjob-official-processors/src/main/java/tech/powerjob/official/processors/util/SecurityUtils.java Normal file
View File

@ -0,0 +1,24 @@
package tech.powerjob.official.processors.util;
import org.apache.commons.lang3.StringUtils;
/**
* Some dangerous processors must be passed in the specified JVM startup parameters to be enabled
*
* @author tjq
* @since 2021/3/14
*/
public class SecurityUtils {
public static final String ENABLE_FILE_CLEANUP_PROCESSOR = "powerjob.official-processor.file-cleanup.enable";
public static final String ENABLE_DYNAMIC_SQL_PROCESSOR = "powerjob.official-processor.dynamic-datasource.enable";
public static boolean disable(String dKey) {
if (StringUtils.isEmpty(dKey)) {
return false;
}
String property = System.getProperty(dKey);
return !StringUtils.equals(Boolean.TRUE.toString(), property);
}
}

32
powerjob-official-processors/src/test/java/tech/powerjob/official/processors/TestUtils.java Normal file
View File

@ -0,0 +1,32 @@
package tech.powerjob.official.processors;
import tech.powerjob.common.model.LogConfig;
import tech.powerjob.worker.core.processor.TaskContext;
import tech.powerjob.worker.core.processor.WorkflowContext;
import tech.powerjob.worker.log.impl.OmsLocalLogger;
import java.util.concurrent.ThreadLocalRandom;
/**
* TestUtils
*
* @author tjq
* @since 2021/1/30
*/
public class TestUtils {
public static TaskContext genTaskContext(String jobParams) {
long jobId = ThreadLocalRandom.current().nextLong();
TaskContext taskContext = new TaskContext();
taskContext.setJobId(jobId);
taskContext.setInstanceId(jobId);
taskContext.setJobParams(jobParams);
taskContext.setTaskId("0.0");
taskContext.setTaskName("TEST_TASK");
taskContext.setOmsLogger(new OmsLocalLogger(new LogConfig()));
taskContext.setWorkflowContext(new WorkflowContext(null, null));
return taskContext;
}
}

65
powerjob-official-processors/src/test/java/tech/powerjob/official/processors/impl/FileCleanupProcessorTest.java Normal file
View File

@ -0,0 +1,65 @@
package tech.powerjob.official.processors.impl;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import tech.powerjob.worker.core.processor.TaskContext;
import org.junit.jupiter.api.Test;
import tech.powerjob.official.processors.TestUtils;
import java.util.regex.Pattern;
import static org.junit.jupiter.api.Assertions.*;
/**
* test FileCleanupProcessor
*
* @author tjq
* @since 2021/2/1
*/
class FileCleanupProcessorTest {
@Test
void testPatternCompile() throws Exception {
String fileName = "abc.log";
System.out.println(fileName.matches("[\\s\\S]*log"));
System.out.println(Pattern.matches("[a-z.0-9]*log", fileName));
}
@Test
void testScriptCompile() throws Exception {
Pattern compile = Pattern.compile("(shell|python)_[0-9]*\\.(sh|py)");
String fileNameA = "shell_158671537124147264.sh";
String fileNameB = "python_158671537124147264.py";
assertTrue(compile.matcher(fileNameA).matches());
assertTrue(compile.matcher(fileNameB).matches());
}
@Test
void testProcess() throws Exception {
JSONObject params = new JSONObject();
params.put("dirPath", "/Users/tjq/logs");
params.put("filePattern", "[\\s\\S]*log");
params.put("retentionTime", 0);
JSONArray array = new JSONArray();
array.add(params);
String paramsStr = array.toJSONString();
System.out.println(paramsStr);
TaskContext taskContext = TestUtils.genTaskContext(paramsStr);
System.out.println(new FileCleanupProcessor().process(taskContext));
}
@Test
void testCleanWorkerScript() throws Exception {
JSONObject params = new JSONObject();
params.put("dirPath", "/");
params.put("filePattern", "(shell|python)_[0-9]*\\.(sh|py)");
params.put("retentionTime", 24);
JSONArray array = new JSONArray();
array.add(params);
TaskContext taskContext = TestUtils.genTaskContext(array.toJSONString());
System.out.println(new FileCleanupProcessor().process(taskContext));
}
}

73
powerjob-official-processors/src/test/java/tech/powerjob/official/processors/impl/HttpProcessorTest.java Normal file
View File

@ -0,0 +1,73 @@
package tech.powerjob.official.processors.impl;
import com.alibaba.fastjson.JSONObject;
import org.junit.jupiter.api.Test;
import tech.powerjob.official.processors.TestUtils;
/**
* HttpProcessorTest
*
* @author tjq
* @since 2021/1/31
*/
class HttpProcessorTest {
@Test
void testDefaultMethod() throws Exception {
String url = "https://www.baidu.com";
JSONObject params = new JSONObject();
params.put("url", url);
System.out.println(new HttpProcessor().process(TestUtils.genTaskContext(params.toJSONString())));
}
@Test
void testGet() throws Exception {
String url = "https://www.baidu.com";
JSONObject params = new JSONObject();
params.put("url", url);
params.put("method", "GET");
System.out.println(new HttpProcessor().process(TestUtils.genTaskContext(params.toJSONString())));
}
@Test
void testPost() throws Exception {
String url = "https://mock.uutool.cn/4f5qfgcdahj0?test=true";
JSONObject params = new JSONObject();
params.put("url", url);
params.put("method", "POST");
params.put("mediaType", "application/json");
params.put("body", params.toJSONString());
System.out.println(new HttpProcessor().process(TestUtils.genTaskContext(params.toJSONString())));
}
@Test
void testPostDefaultJson() throws Exception {
String url = "https://mock.uutool.cn/4f5qfgcdahj0?test=true";
JSONObject params = new JSONObject();
params.put("url", url);
params.put("method", "POST");
System.out.println(new HttpProcessor().process(TestUtils.genTaskContext(params.toJSONString())));
}
@Test
void testPostDefaultWithMediaType() throws Exception {
String url = "https://mock.uutool.cn/4f5qfgcdahj0?test=true";
JSONObject params = new JSONObject();
params.put("url", url);
params.put("method", "POST");
params.put("mediaType", "application/json");
System.out.println(new HttpProcessor().process(TestUtils.genTaskContext(params.toJSONString())));
}
@Test
void testTimeout() throws Exception {
String url = "http://localhost:7700/tmp/sleep";
JSONObject params = new JSONObject();
params.put("url", url);
params.put("method", "GET");
params.put("timeout", 20);
System.out.println(new HttpProcessor().process(TestUtils.genTaskContext(params.toJSONString())));
}
}

48
powerjob-official-processors/src/test/java/tech/powerjob/official/processors/impl/sql/DynamicDatasourceSqlProcessorTest.java Normal file
View File

@ -0,0 +1,48 @@
package tech.powerjob.official.processors.impl.sql;
import com.alibaba.fastjson.JSONObject;
import org.junit.jupiter.api.Test;
import tech.powerjob.official.processors.TestUtils;
import tech.powerjob.official.processors.util.SecurityUtils;
import tech.powerjob.worker.core.processor.ProcessResult;
import tech.powerjob.worker.core.processor.TaskContext;
import static org.junit.jupiter.api.Assertions.*;
/**
* @author tjq
* @since 2021/3/14
*/
class DynamicDatasourceSqlProcessorTest {
@Test
void testSecurity() throws Exception {
ProcessResult ps = new DynamicDatasourceSqlProcessor().process(genDynamicSqlCtx());
assertFalse(ps.isSuccess());
}
@Test
void testProcess() throws Exception {
System.setProperty(SecurityUtils.ENABLE_DYNAMIC_SQL_PROCESSOR, Boolean.TRUE.toString());
ProcessResult ps = new DynamicDatasourceSqlProcessor().process(genDynamicSqlCtx());
assertTrue(ps.isSuccess());
}
private static TaskContext genDynamicSqlCtx() {
JSONObject params = new JSONObject();
// connection info
params.put("jdbcUrl", "jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai");
params.put("user", "root");
params.put("password", "No1Bug2Please3!");
params.put("sql", "select * from job_info");
params.put("showResult", true);
String jobParams = params.toJSONString();
System.out.println(jobParams);
return TestUtils.genTaskContext(jobParams);
}
}

115
powerjob-official-processors/src/test/java/tech/powerjob/official/processors/impl/sql/SpringDatasourceSqlProcessorTest.java Normal file
View File

@ -0,0 +1,115 @@
package tech.powerjob.official.processors.impl.sql;
import com.alibaba.fastjson.JSON;
import tech.powerjob.worker.core.processor.ProcessResult;
import lombok.extern.slf4j.Slf4j;
import org.h2.jdbc.JdbcSQLIntegrityConstraintViolationException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabase;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder;
import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType;
import tech.powerjob.official.processors.TestUtils;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* @author Echo009
* @since 2021/3/11
*/
@Slf4j
class SpringDatasourceSqlProcessorTest {
private static SpringDatasourceSqlProcessor springDatasourceSqlProcessor;
@BeforeAll
static void initSqlProcessor() {
EmbeddedDatabaseBuilder builder = new EmbeddedDatabaseBuilder();
EmbeddedDatabase database = builder.setType(EmbeddedDatabaseType.H2)
.addScript("classpath:db_init.sql")
.build();
springDatasourceSqlProcessor = new SpringDatasourceSqlProcessor(database);
// do nothing
springDatasourceSqlProcessor.registerSqlValidator("fakeSqlValidator", (sql) -> true);
// 排除掉包含 drop 的 SQL
springDatasourceSqlProcessor.registerSqlValidator("interceptDropValidator", (sql) -> sql.matches("^(?i)((?!drop).)*$"));
// add ';'
springDatasourceSqlProcessor.setSqlParser((sql, taskContext) -> {
if (!sql.endsWith(";")) {
return sql + ";";
}
return sql;
});
// just invoke clean datasource method
springDatasourceSqlProcessor.removeDataSource("NULL_DATASOURCE");
log.info("init sql processor successfully!");
}
@Test
void testSqlValidator() {
SpringDatasourceSqlProcessor.SqlParams sqlParams = new SpringDatasourceSqlProcessor.SqlParams();
sqlParams.setSql("drop table test_table");
// 校验不通过
assertThrows(IllegalArgumentException.class, () -> springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams))));
}
@Test
void testIncorrectDataSourceName() {
SpringDatasourceSqlProcessor.SqlParams sqlParams = constructSqlParam("create table task_info (a varchar(255), b varchar(255), c varchar(255))");
sqlParams.setDataSourceName("(๑•̀ㅂ•́)و✧");
// 数据源名称非法
assertThrows(IllegalArgumentException.class, () -> springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams))));
}
@Test
void testExecDDL() {
SpringDatasourceSqlProcessor.SqlParams sqlParams = constructSqlParam("create table power_job (a varchar(255), b varchar(255), c varchar(255))");
ProcessResult processResult = springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams)));
assertTrue(processResult.isSuccess());
}
@Test
void testExecSQL() {
SpringDatasourceSqlProcessor.SqlParams sqlParams1 = constructSqlParam("insert into test_table (id, content) values (0, 'Fight for a better tomorrow')");
ProcessResult processResult1 = springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams1)));
assertTrue(processResult1.isSuccess());
assertThrows(JdbcSQLIntegrityConstraintViolationException.class, () -> springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams1))));
// 第二条会失败回滚
SpringDatasourceSqlProcessor.SqlParams sqlParams2 = constructSqlParam("insert into test_table (id, content) values (1, '?');insert into test_table (id, content) values (0, 'Fight for a better tomorrow')");
assertThrows(JdbcSQLIntegrityConstraintViolationException.class, () -> springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams2))));
// 上方回滚，这里就能成功插入
SpringDatasourceSqlProcessor.SqlParams sqlParams3 = constructSqlParam("insert into test_table (id, content) values (1, '?')");
ProcessResult processResult3 = springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams3)));
assertTrue(processResult3.isSuccess());
SpringDatasourceSqlProcessor.SqlParams sqlParams4 = constructSqlParam("insert into test_table (id, content) values (2, '?');insert into test_table (id, content) values (3, '?')");
ProcessResult processResult4 = springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(sqlParams4)));
assertTrue(processResult4.isSuccess());
}
@Test
public void testQuery() {
SpringDatasourceSqlProcessor.SqlParams insertParams = constructSqlParam("insert into test_table (id, content) values (1, '?');insert into test_table (id, content) values (0, 'Fight for a better tomorrow')");
springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(insertParams)));
SpringDatasourceSqlProcessor.SqlParams queryParams = constructSqlParam("select * from test_table");
springDatasourceSqlProcessor.process0(TestUtils.genTaskContext(JSON.toJSONString(queryParams)));
}
static SpringDatasourceSqlProcessor.SqlParams constructSqlParam(String sql){
SpringDatasourceSqlProcessor.SqlParams sqlParams = new SpringDatasourceSqlProcessor.SqlParams();
sqlParams.setSql(sql);
sqlParams.setShowResult(true);
return sqlParams;
}
}

7
powerjob-official-processors/src/test/resources/db_init.sql Normal file
View File

@ -0,0 +1,7 @@
create table test_table
(
id bigint(20) primary key,
content varchar(255),
gmt_create datetime default now(),
gmt_modified datetime default now()
);