geg-gas/powerjob-kingbase
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

---初始化项目

Browse Source
This commit is contained in:
张福财
2025-09-19 16:14:08 +08:00
parent 902d3d7e3b
commit afee7c03ac
767 changed files with 75809 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
powerjob-server/powerjob-server-persistence/pom.xml Normal file
View File

@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>powerjob-server</artifactId>
<groupId>tech.powerjob</groupId>
<version>5.1.2</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>powerjob-server-persistence</artifactId>
<version>${project.parent.version}</version>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>tech.powerjob</groupId>
<artifactId>powerjob-server-common</artifactId>
</dependency>
<dependency>
<groupId>tech.powerjob</groupId>
<artifactId>powerjob-server-extension</artifactId>
</dependency>
<dependency>
<groupId>tech.powerjob</groupId>
<artifactId>powerjob-server-monitor</artifactId>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-sync</artifactId>
</dependency>
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
</dependency>
</dependencies>
</project>

47
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/PageResult.java Normal file
View File

@ -0,0 +1,47 @@
package tech.powerjob.server.persistence;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.domain.Page;
import java.io.Serializable;
import java.util.List;
/**
* 分页对象
*
* @author tjq
* @since 2020/4/12
*/
@Data
@NoArgsConstructor
public class PageResult<T> implements Serializable {
/**
* 当前页数
*/
private int index;
/**
* 页大小
*/
private int pageSize;
/**
* 总页数
*/
private int totalPages;
/**
* 总数据量
*/
private long totalItems;
/**
* 数据
*/
private List<T> data;
public PageResult(Page<?> page) {
index = page.getNumber();
pageSize = page.getSize();
totalPages = page.getTotalPages();
totalItems = page.getTotalElements();
}
}

128
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/QueryConvertUtils.java Normal file
View File

@ -0,0 +1,128 @@
package tech.powerjob.server.persistence;
import com.alibaba.fastjson.JSONArray;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.domain.Specification;
import tech.powerjob.common.PowerQuery;
import tech.powerjob.common.exception.PowerJobException;
import tech.powerjob.common.request.query.PowerPageQuery;
import javax.persistence.criteria.Predicate;
import java.lang.reflect.Field;
import java.util.List;
/**
* auto convert query to Specification
*
* @author tjq
* @since 2021/1/15
*/
@Slf4j
@SuppressWarnings("unchecked, rawtypes")
public class QueryConvertUtils {
public static <T> Specification<T> toSpecification(PowerQuery powerQuery) {
return (Specification<T>) (root, query, cb) -> {
List<Predicate> predicates = Lists.newLinkedList();
Field[] fields = powerQuery.getClass().getDeclaredFields();
try {
for (Field field : fields) {
field.setAccessible(true);
String fieldName = field.getName();
Object fieldValue = field.get(powerQuery);
if (fieldValue == null) {
continue;
}
if (fieldName.endsWith(PowerQuery.EQUAL)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.EQUAL);
predicates.add(cb.equal(root.get(colName), fieldValue));
} else if (fieldName.endsWith(PowerQuery.NOT_EQUAL)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.NOT_EQUAL);
predicates.add(cb.notEqual(root.get(colName), fieldValue));
} else if (fieldName.endsWith(PowerQuery.LIKE)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.LIKE);
predicates.add(cb.like(root.get(colName), convertLikeParams(fieldValue)));
} else if (fieldName.endsWith(PowerQuery.NOT_LIKE)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.NOT_LIKE);
predicates.add(cb.notLike(root.get(colName), convertLikeParams(fieldValue)));
} else if (fieldName.endsWith(PowerQuery.LESS_THAN)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.LESS_THAN);
predicates.add(cb.lessThan(root.get(colName), (Comparable)fieldValue));
} else if (fieldName.endsWith(PowerQuery.GREATER_THAN)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.GREATER_THAN);
predicates.add(cb.greaterThan(root.get(colName), (Comparable)fieldValue));
} else if (fieldName.endsWith(PowerQuery.LESS_THAN_EQUAL)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.LESS_THAN_EQUAL);
predicates.add(cb.lessThanOrEqualTo(root.get(colName), (Comparable)fieldValue));
} else if (fieldName.endsWith(PowerQuery.GREATER_THAN_EQUAL)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.GREATER_THAN_EQUAL);
predicates.add(cb.greaterThanOrEqualTo(root.get(colName), (Comparable)fieldValue));
} else if (fieldName.endsWith(PowerQuery.IN)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.IN);
predicates.add(root.get(colName).in(convertInParams(fieldValue)));
} else if (fieldName.endsWith(PowerQuery.NOT_IN)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.NOT_IN);
predicates.add(cb.not(root.get(colName).in(convertInParams(fieldValue))));
} else if (fieldName.endsWith(PowerQuery.IS_NULL)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.IS_NULL);
predicates.add(cb.isNull(root.get(colName)));
} else if (fieldName.endsWith(PowerQuery.IS_NOT_NULL)) {
String colName = StringUtils.substringBeforeLast(fieldName, PowerQuery.IS_NOT_NULL);
predicates.add(cb.isNotNull(root.get(colName)));
}
}
} catch (Exception e) {
log.warn("[QueryConvertUtils] convert failed for query: {}", query, e);
throw new PowerJobException("convert query object failed, maybe you should redesign your query object!");
}
if (powerQuery.getAppIdEq() != null) {
predicates.add(cb.equal(root.get("appId"), powerQuery.getAppIdEq()));
}
return query.where(predicates.toArray(new Predicate[0])).getRestriction();
};
}
public static Pageable toPageable(PowerPageQuery powerPageQuery) {
Sort sorter = null;
String sortBy = powerPageQuery.getSortBy();
if (StringUtils.isNoneEmpty(sortBy)) {
sorter = Sort.by(sortBy);
if (powerPageQuery.isAsc()) {
sorter = sorter.ascending();
} else {
sorter = sorter.descending();
}
}
if (sorter == null) {
return PageRequest.of(powerPageQuery.getIndex(), powerPageQuery.getPageSize());
}
return PageRequest.of(powerPageQuery.getIndex(), powerPageQuery.getPageSize(), sorter);
}
public static String convertLikeParams(Object o) {
String s = (String) o;
if (!s.startsWith("%")) {
s = "%" + s;
}
if (!s.endsWith("%")) {
s = s + "%";
}
return s;
}
private static Object[] convertInParams(Object o) {
// FastJSON, 永远滴神！
return JSONArray.parseArray(JSONArray.toJSONString(o)).toArray();
}
}

37
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/StringPage.java Normal file
View File

@ -0,0 +1,37 @@
package tech.powerjob.server.persistence;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* 文本分页
*
* @author tjq
* @since 2020/5/3
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class StringPage {
/**
* 当前页数
*/
private long index;
/**
* 总页数
*/
private long totalPages;
/**
* 文本数据
*/
private String data;
public static StringPage simple(String data) {
StringPage sp = new StringPage();
sp.index = 0;
sp.totalPages = 1;
sp.data = data;
return sp;
}
}

78
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/config/LocalJpaConfig.java Normal file
View File

@ -0,0 +1,78 @@
package tech.powerjob.server.persistence.config;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateProperties;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateSettings;
import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.TransactionDefinition;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import org.springframework.transaction.support.TransactionTemplate;
import javax.annotation.Resource;
import javax.sql.DataSource;
import java.util.Map;
import java.util.Objects;
/**
* 本地H2数据库配置
*
* @author tjq
* @since 2020/4/27
*/
@Configuration
@EnableTransactionManagement
@EnableJpaRepositories(
// repository包名
basePackages = LocalJpaConfig.LOCAL_PACKAGES,
// 实体管理bean名称
entityManagerFactoryRef = "localEntityManagerFactory",
// 事务管理bean名称
transactionManagerRef = "localTransactionManager"
)
public class LocalJpaConfig {
public static final String LOCAL_PACKAGES = "tech.powerjob.server.persistence.local";
private static Map<String, Object> genDatasourceProperties() {
JpaProperties jpaProperties = new JpaProperties();
jpaProperties.setOpenInView(false);
jpaProperties.setShowSql(false);
HibernateProperties hibernateProperties = new HibernateProperties();
// 每次启动都删除数据（重启后原来的Instance已经通过故障转移更换了Server，老的日志数据也没什么意义了）
hibernateProperties.setDdlAuto("create");
return hibernateProperties.determineHibernateProperties(jpaProperties.getProperties(), new HibernateSettings());
}
@Bean(name = "localEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean initLocalEntityManagerFactory(@Qualifier("omsLocalDatasource") DataSource omsLocalDatasource,EntityManagerFactoryBuilder builder) {
return builder
.dataSource(omsLocalDatasource)
.properties(genDatasourceProperties())
.packages(LOCAL_PACKAGES)
.persistenceUnit("localPersistenceUnit")
.build();
}
@Bean(name = "localTransactionManager")
public PlatformTransactionManager initLocalTransactionManager(@Qualifier("localEntityManagerFactory") LocalContainerEntityManagerFactoryBean localContainerEntityManagerFactoryBean) {
return new JpaTransactionManager(Objects.requireNonNull(localContainerEntityManagerFactoryBean.getObject()));
}
@Bean(name = "localTransactionTemplate")
public TransactionTemplate initTransactionTemplate(@Qualifier("localTransactionManager") PlatformTransactionManager ptm) {
TransactionTemplate tt = new TransactionTemplate(ptm);
// 设置隔离级别
tt.setIsolationLevel(TransactionDefinition.ISOLATION_DEFAULT);
return tt;
}
}

56
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/config/MultiDatasourceConfig.java Normal file
View File

@ -0,0 +1,56 @@
package tech.powerjob.server.persistence.config;
import tech.powerjob.server.common.utils.OmsFileUtils;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import org.apache.commons.io.FileUtils;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import javax.sql.DataSource;
import java.io.File;
/**
* 多重数据源配置
*
* @author tjq
* @since 2020/4/27
*/
@Configuration
public class MultiDatasourceConfig {
private static final String H2_DRIVER_CLASS_NAME = "org.h2.Driver";
private static final String H2_JDBC_URL_PATTERN = "jdbc:h2:file:%spowerjob_server_db";
private static final int H2_MIN_SIZE = 4;
private static final int H2_MAX_ACTIVE_SIZE = 10;
@Primary
@Bean("omsRemoteDatasource")
@ConfigurationProperties(prefix = "spring.datasource.core")
public DataSource initOmsCoreDatasource() {
return DataSourceBuilder.create().build();
}
@Bean("omsLocalDatasource")
public DataSource initOmsLocalDatasource() {
String h2Path = OmsFileUtils.genH2WorkPath();
HikariConfig config = new HikariConfig();
config.setDriverClassName(H2_DRIVER_CLASS_NAME);
config.setJdbcUrl(String.format(H2_JDBC_URL_PATTERN, h2Path));
config.setAutoCommit(true);
// 池中最小空闲连接数量
config.setMinimumIdle(H2_MIN_SIZE);
// 池中最大连接数量
config.setMaximumPoolSize(H2_MAX_ACTIVE_SIZE);
// JVM 关闭时删除文件
try {
FileUtils.forceDeleteOnExit(new File(h2Path));
}catch (Exception ignore) {
}
return new HikariDataSource(config);
}
}

66
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/config/MultiDatasourceProperties.java Normal file
View File

@ -0,0 +1,66 @@
package tech.powerjob.server.persistence.config;
import com.google.common.collect.Maps;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.*;
/**
* 多重数据源配置
*
* @author Kung Yao
* @since 2020/4/27
*/
@Component
@ConfigurationProperties("spring.datasource")
public class MultiDatasourceProperties {
private DataSourceProperties remote = new DataSourceProperties();
private DataSourceProperties local = new DataSourceProperties();
public static class DataSourceProperties {
private HibernateProperties hibernate = new HibernateProperties();
public void setHibernate(HibernateProperties hibernate) {
this.hibernate = hibernate;
}
public HibernateProperties getHibernate() {
return hibernate;
}
}
public static class HibernateProperties {
private Map<String, String> properties = Maps.newHashMap();
public void setProperties(Map<String, String> properties) {
this.properties = properties;
}
public Map<String, String> getProperties() {
return properties;
}
}
public void setLocal(DataSourceProperties local) {
this.local = local;
}
public void setRemote(DataSourceProperties remote) {
this.remote = remote;
}
public DataSourceProperties getLocal() {
return local;
}
public DataSourceProperties getRemote() {
return remote;
}
}

47
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/config/PowerJobPhysicalNamingStrategy.java Normal file
View File

@ -0,0 +1,47 @@
package tech.powerjob.server.persistence.config;
import tech.powerjob.server.common.PowerJobServerConfigKey;
import tech.powerjob.server.common.utils.PropertyUtils;
import org.hibernate.boot.model.naming.Identifier;
import org.hibernate.engine.jdbc.env.spi.JdbcEnvironment;
import org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy;
import org.apache.commons.lang3.StringUtils;
import java.io.Serializable;
/**
* 自定义表前缀，配置项 oms.table-prefix 不配置时，不增加表前缀。
* 参考实现：{@link org.springframework.boot.orm.jpa.hibernate.SpringPhysicalNamingStrategy}
* <p>
* 1. 继承 PhysicalNamingStrategy 类，实现自定义表前缀；
* </p>
* <p>
* 2. 修改@Query(nativeQuery = true)和其SQL，用对象名和属性名代替表名和数据库字段名。
* </p>
*
* @author songyinyin
* @since 2020/7/18
*/
public class PowerJobPhysicalNamingStrategy extends SpringPhysicalNamingStrategy implements Serializable {
/**
* 映射物理表名称，如：把实体表 AppInfoDO 的 DO 去掉，再加上表前缀
*
* @param name 实体名称
* @param jdbcEnvironment jdbc环境变量
* @return 映射后的物理表
*/
@Override
public Identifier toPhysicalTableName(Identifier name, JdbcEnvironment jdbcEnvironment) {
String tablePrefix = PropertyUtils.getProperties().getProperty(PowerJobServerConfigKey.TABLE_PREFIX);
String text = name.getText();
String noDOText = StringUtils.endsWithIgnoreCase(text, "do") ? text.substring(0, text.length() - 2) : text;
String newText = StringUtils.isNotEmpty(tablePrefix) ? tablePrefix + noDOText : noDOText;
return super.toPhysicalTableName(new Identifier(newText, name.isQuoted()), jdbcEnvironment);
}
}

83
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/config/RemoteJpaConfig.java Normal file
View File

@ -0,0 +1,83 @@
package tech.powerjob.server.persistence.config;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateProperties;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateSettings;
import org.springframework.boot.autoconfigure.orm.jpa.JpaProperties;
import org.springframework.boot.orm.jpa.EntityManagerFactoryBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.orm.jpa.JpaTransactionManager;
import org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean;
import org.springframework.transaction.PlatformTransactionManager;
import org.springframework.transaction.annotation.EnableTransactionManagement;
import javax.sql.DataSource;
import java.util.Map;
import java.util.Objects;
/**
* 核心数据库 JPA 配置
*
* @author tjq
* @since 2020/4/27
*/
@Configuration
@EnableTransactionManagement
@EnableJpaRepositories(
// repository包名
basePackages = RemoteJpaConfig.CORE_PACKAGES,
// 实体管理bean名称
entityManagerFactoryRef = "remoteEntityManagerFactory",
// 事务管理bean名称
transactionManagerRef = "remoteTransactionManager"
)
public class RemoteJpaConfig {
public static final String CORE_PACKAGES = "tech.powerjob.server.persistence.remote";
/**
* 生成配置文件，包括 JPA配置文件和Hibernate配置文件，相当于以下三个配置
* spring.jpa.show-sql=false
* spring.jpa.open-in-view=false
* spring.jpa.hibernate.ddl-auto=update
*
* @return 配置Map
*/
private static Map<String, Object> genDatasourceProperties() {
JpaProperties jpaProperties = new JpaProperties();
jpaProperties.setOpenInView(false);
jpaProperties.setShowSql(false);
HibernateProperties hibernateProperties = new HibernateProperties();
hibernateProperties.setDdlAuto("update");
// 配置JPA自定义表名称策略
hibernateProperties.getNaming().setPhysicalStrategy(PowerJobPhysicalNamingStrategy.class.getName());
HibernateSettings hibernateSettings = new HibernateSettings();
return hibernateProperties.determineHibernateProperties(jpaProperties.getProperties(), hibernateSettings);
}
@Primary
@Bean(name = "remoteEntityManagerFactory")
public LocalContainerEntityManagerFactoryBean initRemoteEntityManagerFactory(@Qualifier("omsRemoteDatasource") DataSource omsRemoteDatasource,@Qualifier("multiDatasourceProperties") MultiDatasourceProperties properties, EntityManagerFactoryBuilder builder) {
Map<String, Object> datasourceProperties = genDatasourceProperties();
datasourceProperties.putAll(properties.getRemote().getHibernate().getProperties());
return builder
.dataSource(omsRemoteDatasource)
.properties(datasourceProperties)
.packages(CORE_PACKAGES)
.persistenceUnit("remotePersistenceUnit")
.build();
}
@Primary
@Bean(name = "remoteTransactionManager")
public PlatformTransactionManager initRemoteTransactionManager(@Qualifier("remoteEntityManagerFactory") LocalContainerEntityManagerFactoryBean localContainerEntityManagerFactoryBean) {
return new JpaTransactionManager(Objects.requireNonNull(localContainerEntityManagerFactoryBean.getObject()));
}
}

37
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/config/dialect/AdpPostgreSQLDialect.java Normal file
View File

@ -0,0 +1,37 @@
package tech.powerjob.server.persistence.config.dialect;
import org.hibernate.dialect.PostgreSQL10Dialect;
import org.hibernate.type.descriptor.sql.LongVarbinaryTypeDescriptor;
import org.hibernate.type.descriptor.sql.LongVarcharTypeDescriptor;
import org.hibernate.type.descriptor.sql.SqlTypeDescriptor;
import java.sql.Types;
/**
* <a href="https://github.com/PowerJob/PowerJob/issues/750">PG数据库方言</a>
* 使用方自行通过配置文件激活：spring.datasource.remote.hibernate.properties.hibernate.dialect=tech.powerjob.server.persistence.config.dialect.AdpPostgreSQLDialect
*
* @author litong0531
* @since 2024/8/11
*/
public class AdpPostgreSQLDialect extends PostgreSQL10Dialect {
public AdpPostgreSQLDialect() {
super();
registerColumnType(Types.BLOB, "bytea");
registerColumnType(Types.CLOB, "text");
}
@Override
public SqlTypeDescriptor remapSqlTypeDescriptor(SqlTypeDescriptor sqlTypeDescriptor) {
switch (sqlTypeDescriptor.getSqlType()) {
case Types.CLOB:
return LongVarcharTypeDescriptor.INSTANCE;
case Types.BLOB:
return LongVarbinaryTypeDescriptor.INSTANCE;
case Types.NCLOB:
return LongVarbinaryTypeDescriptor.INSTANCE;
}
return super.remapSqlTypeDescriptor(sqlTypeDescriptor);
}
}

33
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/config/dialect/PowerJobPGDialect.java Normal file
View File

@ -0,0 +1,33 @@
package tech.powerjob.server.persistence.config.dialect;
import org.hibernate.dialect.PostgreSQL10Dialect;
import org.hibernate.type.descriptor.sql.LongVarcharTypeDescriptor;
import org.hibernate.type.descriptor.sql.SqlTypeDescriptor;
import java.sql.Types;
/**
* PostgreSQL 数据库支持，需要在 application.properties 中添加以下配置项进行激活
* spring.datasource.remote.hibernate.properties.hibernate.dialect=tech.powerjob.server.persistence.config.dialect.PowerJobPGDialect
*
* @author Kung Yao
* @author Echo009
* 2021/3/24 下午 04:23
* 1074_King
*/
public class PowerJobPGDialect extends PostgreSQL10Dialect {
/**
* 使用 {@link Types#LONGVARCHAR} 覆盖 {@link Types#CLOB} 类型
*
* 注意，如果在 PG 库创建表时使用的列类型为 oid ，那么这样会导致没法正确读取数据
* 在 PowerJob 中能这样用是因为 PowerJob 的所有实体类中被 @Lob 注解标记的列对应数据库中的字段类型都是 text
* 另外还需要注意数据库版本，如果是 10.x 以前的，需自行提供一个合适的 Dialect 类（选择合适的版本继承）
*
* 更多内容请关注该 issues：https://github.com/PowerJob/PowerJob/issues/153
*/
@Override
public SqlTypeDescriptor getSqlTypeDescriptorOverride(int sqlCode) {
return Types.CLOB == sqlCode ? LongVarcharTypeDescriptor.INSTANCE : null;
}
}

47
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/local/LocalInstanceLogDO.java Normal file
View File

@ -0,0 +1,47 @@
package tech.powerjob.server.persistence.local;
import tech.powerjob.common.enums.LogLevel;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.persistence.*;
/**
* 本地的运行时日志
*
* @author tjq
* @since 2020/4/27
*/
@Data
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(name = "local_instance_log", indexes = {@Index(columnList = "instanceId")})
public class LocalInstanceLogDO {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
private Long instanceId;
/**
* 日志时间
*/
private Long logTime;
/**
* 日志级别 {@link LogLevel}
*/
private Integer logLevel;
/**
* 日志内容
*/
@Lob
@Column(columnDefinition="TEXT")
private String logContent;
/**
* 机器地址
*/
private String workerAddress;
}

37
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/local/LocalInstanceLogRepository.java Normal file
View File

@ -0,0 +1,37 @@
package tech.powerjob.server.persistence.local;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import javax.transaction.Transactional;
import java.util.List;
import java.util.stream.Stream;
/**
* 本地运行时日志数据操作层
*
* @author tjq
* @since 2020/4/27
*/
public interface LocalInstanceLogRepository extends JpaRepository<LocalInstanceLogDO, Long> {
/**
* 流式查询
*/
Stream<LocalInstanceLogDO> findByInstanceIdOrderByLogTime(Long instanceId);
/**
* 删除数据
*/
@Modifying
@Transactional(rollbackOn = Exception.class)
long deleteByInstanceId(Long instanceId);
@Modifying
@Transactional(rollbackOn = Exception.class)
@CanIgnoreReturnValue
long deleteByInstanceIdInAndLogTimeLessThan(List<Long> instanceIds, Long t);
long countByInstanceId(Long instanceId);
}

90
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/monitor/DatabaseMonitorAspect.java Normal file
View File

@ -0,0 +1,90 @@
package tech.powerjob.server.persistence.monitor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.springframework.data.domain.Slice;
import org.springframework.stereotype.Component;
import tech.powerjob.server.common.utils.AOPUtils;
import tech.powerjob.server.monitor.MonitorService;
import tech.powerjob.server.monitor.events.db.DatabaseEvent;
import tech.powerjob.server.monitor.events.db.DatabaseType;
import java.util.Collection;
import java.util.Optional;
import java.util.stream.Stream;
/**
* 监控切面
*
* @author tjq
* @since 2022/9/6
*/
@Slf4j
@Aspect
@Component
@RequiredArgsConstructor
public class DatabaseMonitorAspect {
private final MonitorService monitorService;
@Around("execution(* tech.powerjob.server.persistence.remote.repository..*.*(..))")
public Object monitorCoreDB(ProceedingJoinPoint joinPoint) throws Throwable {
return wrapperMonitor(joinPoint, DatabaseType.CORE);
}
@Around("execution(* tech.powerjob.server.persistence.local..*.*(..))")
public Object monitorLocalDB(ProceedingJoinPoint joinPoint) throws Throwable {
return wrapperMonitor(joinPoint, DatabaseType.LOCAL);
}
private Object wrapperMonitor(ProceedingJoinPoint point, DatabaseType type) throws Throwable {
String classNameMini = AOPUtils.parseRealClassName(point);
final String methodName = point.getSignature().getName();
DatabaseEvent event = new DatabaseEvent().setType(type)
.setServiceName(classNameMini)
.setMethodName(methodName)
.setStatus(DatabaseEvent.Status.SUCCESS);
long startTs = System.currentTimeMillis();
try {
final Object ret = point.proceed();
event.setRows(parseEffectRows(ret));
return ret;
} catch (Throwable t) {
event.setErrorMsg(t.getClass().getSimpleName()).setStatus(DatabaseEvent.Status.FAILED);
throw t;
} finally {
long cost = System.currentTimeMillis() - startTs;
monitorService.monitor(event.setCost(cost));
}
}
private static Integer parseEffectRows(Object ret) {
// 从性能角度考虑，最高频场景放在最前面判断
if (ret instanceof Number) {
return ((Number) ret).intValue();
}
if (ret instanceof Optional) {
return ((Optional<?>) ret).isPresent() ? 1 : 0;
}
if (ret instanceof Collection) {
return ((Collection<?>) ret).size();
}
if (ret instanceof Slice) {
return ((Slice<?>) ret).getSize();
}
if (ret instanceof Stream) {
return null;
}
// TODO: 直接返回对象的方法全部改成 Optional
return ret == null ? 0 : 1;
}
}

65
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/AppInfoDO.java Normal file
View File

@ -0,0 +1,65 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 应用信息表
*
* @author tjq
* @since 2020/3/30
*/
@Data
@Entity
@Table(uniqueConstraints = {@UniqueConstraint(name = "uidx01_app_info", columnNames = {"appName"})})
public class AppInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
private String appName;
/**
* 描述
*/
private String title;
/**
* 应用分组密码
*/
private String password;
/**
* 当前负责该 appName 旗下任务调度的server地址，IP:Port（注意，该地址为ActorSystem地址，而不是HTTP地址，两者端口不同）
* 支持多语言后，尽管引入了 vert.x 的地址，但该字段仍保存 ActorSystem 的地址，vert.x 地址仅在返回给 worker 时特殊处理
* 原因：框架中很多地方强依赖 currentServer，比如根据该地址来获取需要调度的 app
*/
private String currentServer;
/**
* 命名空间ID，外键关联
*/
private Long namespaceId;
/**
* 管理标签
*/
private String tags;
/**
* 扩展字段
*/
private String extra;
private Date gmtCreate;
private Date gmtModified;
private Long creator;
private Long modifier;
}

59
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/ContainerInfoDO.java Normal file
View File

@ -0,0 +1,59 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 容器（jar容器）信息表
*
* @author tjq
* @since 2020/5/15
*/
@Data
@Entity
@Table(indexes = {@Index(name = "idx01_container_info", columnList = "appId")})
public class ContainerInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* 所属的应用ID
*/
private Long appId;
private String containerName;
/**
* 容器类型，枚举值为 ContainerSourceType
*/
private Integer sourceType;
/**
* 由 sourceType 决定，JarFile -> String，存储文件名称；Git -> JSON，包括 URL，branch，username，password
*/
private String sourceInfo;
/**
* 版本 （Jar包使用md5，Git使用commitId，前者32位，后者40位，不会产生碰撞）
*/
private String version;
/**
* 状态，枚举值为 ContainerStatus
*/
private Integer status;
/**
* 上一次部署时间
*/
private Date lastDeployTime;
private Date gmtCreate;
private Date gmtModified;
}

121
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/InstanceInfoDO.java Normal file
View File

@ -0,0 +1,121 @@
package tech.powerjob.server.persistence.remote.model;
import tech.powerjob.common.enums.InstanceStatus;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 任务运行日志表
*
* @author tjq
* @since 2020/3/30
*/
@Data
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(indexes = {
@Index(name = "idx01_instance_info", columnList = "jobId,status"),
@Index(name = "idx02_instance_info", columnList = "appId,status"),
@Index(name = "idx03_instance_info", columnList = "instanceId,status"),
@Index(name = "idx04_instance_info_outer_key", columnList = "outerKey")
})
public class InstanceInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* 任务ID
*/
private Long jobId;
/**
* 任务所属应用的ID，冗余提高查询效率
*/
private Long appId;
/**
* 任务所属应用的ID，冗余提高查询效率
*/
private Long instanceId;
/**
* 任务参数（静态）
*
* @since 2021/2/01
*/
@Lob
@Column
private String jobParams;
/**
* 任务实例参数（动态）
*/
@Lob
@Column
private String instanceParams;
/**
* 该任务实例的类型，普通/工作流（InstanceType）
*/
private Integer type;
/**
* 该任务实例所属的 workflow ID，仅 workflow 任务存在
*/
private Long wfInstanceId;
/**
* 任务状态 {@link InstanceStatus}
*/
private Integer status;
/**
* 执行结果（允许存储稍大的结果）
*/
@Lob
@Column
private String result;
/**
* 预计触发时间
*/
private Long expectedTriggerTime;
/**
* 实际触发时间
*/
private Long actualTriggerTime;
/**
* 结束时间
*/
private Long finishedTime;
/**
* 最后上报时间
*/
private Long lastReportTime;
/**
* TaskTracker 地址
*/
private String taskTrackerAddress;
/**
* 总共执行的次数（用于重试判断）
*/
private Long runningTimes;
/**
* “外键”，用于 OPENAPI 场景业务场景与 PowerJob 实例的绑定
*/
private String outerKey;
/**
* 扩展属性，用于 OPENAPI 场景上下文参数的透传
*/
private String extendValue;
/**
* 调度元信息
*/
private String meta;
private Date gmtCreate;
private Date gmtModified;
}

172
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/JobInfoDO.java Normal file
View File

@ -0,0 +1,172 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 任务信息表
*
* @author tjq
* @since 2020/3/29
*/
@Data
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(indexes = {
@Index(name = "idx01_job_info", columnList = "appId,status,timeExpressionType,nextTriggerTime"),
})
public class JobInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/* ************************** 任务基本信息 ************************** */
/**
* 任务名称
*/
private String jobName;
/**
* 任务描述
*/
private String jobDescription;
/**
* 任务所属的应用ID
*/
private Long appId;
/**
* 任务自带的参数
*/
@Lob
@Column
private String jobParams;
/* ************************** 定时参数 ************************** */
/**
* 时间表达式类型（CRON/API/FIX_RATE/FIX_DELAY）
*/
private Integer timeExpressionType;
/**
* 时间表达式，CRON/NULL/LONG/LONG
*/
private String timeExpression;
/* ************************** 执行方式 ************************** */
/**
* 执行类型，单机/广播/MR
*/
private Integer executeType;
/**
* 执行器类型，Java/Shell
*/
private Integer processorType;
/**
* 执行器信息
*/
private String processorInfo;
/* ************************** 运行时配置 ************************** */
/**
* 最大同时运行任务数，默认 1
*/
private Integer maxInstanceNum;
/**
* 并发度，同时执行某个任务的最大线程数量
*/
private Integer concurrency;
/**
* 任务整体超时时间
*/
private Long instanceTimeLimit;
/* ************************** 重试配置 ************************** */
private Integer instanceRetryNum;
private Integer taskRetryNum;
/**
* 1 正常运行，2 停止（不再调度）
*/
private Integer status;
/**
* 下一次调度时间
*/
private Long nextTriggerTime;
/* ************************** 繁忙机器配置 ************************** */
/**
* 最低CPU核心数量，0代表不限
*/
private double minCpuCores;
/**
* 最低内存空间，单位 GB，0代表不限
*/
private double minMemorySpace;
/**
* 最低磁盘空间，单位 GB，0代表不限
*/
private double minDiskSpace;
/* ************************** 集群配置 ************************** */
/**
* 指定机器运行，空代表不限，非空则只会使用其中的机器运行（多值逗号分割）
*/
private String designatedWorkers;
/**
* 最大机器数量
*/
private Integer maxWorkerCount;
/**
* 报警用户ID列表，多值逗号分隔
*/
private String notifyUserIds;
private Date gmtCreate;
private Date gmtModified;
/**
* 扩展参数，PowerJob 自身不会使用该数据，留给开发者扩展时使用
* 比如 WorkerFilter 的自定义 worker 过滤逻辑，可在此传入过滤指标 GpuUsage < 10
*/
private String extra;
/**
* 派发策略
*/
private Integer dispatchStrategy;
/**
* 某种派发策略背后的具体配置，值取决于 dispatchStrategy
*/
private String dispatchStrategyConfig;
private String lifecycle;
/**
* 告警配置
*/
private String alarmConfig;
/**
* 任务归类，开放给接入方自由定制
*/
private String tag;
/**
* 日志配置，包括日志级别、日志方式等配置信息
*/
private String logConfig;
/**
* 高级运行时配置
* 不需要用于索引的高级运行参数，后续统一存储到这里，便于版本升级（尽可能保证数据库表结构稳定）
*/
private String advancedRuntimeConfig;
}

65
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/NamespaceDO.java Normal file
View File

@ -0,0 +1,65 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 命名空间，用于组织管理 App
*
* @author tjq
* @since 2023/9/3
*/
@Data
@Entity
@Table(uniqueConstraints = {@UniqueConstraint(name = "uidx01_namespace", columnNames = {"code"})})
public class NamespaceDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* 空间唯一标识
*/
private String code;
/**
* 空间名称，比如中文描述（XX部门XX空间）
*/
private String name;
/**
* 鉴权 token，后续 OpenAPI 调用需要
*/
private String token;
private Integer status;
/**
* 部门，组织架构相关属性。
* 预留数据库字段方便基于组织架构二次开发
*/
private String dept;
/**
* 标签，扩展性之王，多值逗号分割
*/
private String tags;
/**
* 扩展字段
*/
private String extra;
private Date gmtCreate;
private Date gmtModified;
private Long creator;
private Long modifier;
}

46
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/OmsLockDO.java Normal file
View File

@ -0,0 +1,46 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 数据库锁
*
* @author tjq
* @since 2020/4/2
*/
@Data
@Entity
@NoArgsConstructor
@Table(uniqueConstraints = {@UniqueConstraint(name = "uidx01_oms_lock", columnNames = {"lockName"})})
public class OmsLockDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
private String lockName;
private String ownerIP;
/**
* 最长持有锁的时间
*/
private Long maxLockTime;
private Date gmtCreate;
private Date gmtModified;
public OmsLockDO(String lockName, String ownerIP, Long maxLockTime) {
this.lockName = lockName;
this.ownerIP = ownerIP;
this.maxLockTime = maxLockTime;
this.gmtCreate = new Date();
this.gmtModified = this.gmtCreate;
}
}

36
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/PwjbUserInfoDO.java Normal file
View File

@ -0,0 +1,36 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* PowerJob 自建登录体系的用户表，只存储使用 PowerJob 自带登录方式登录的用户信息
*
* @author tjq
* @since 2024/2/13
*/
@Data
@Entity
@Table(uniqueConstraints = {
@UniqueConstraint(name = "uidx01_username", columnNames = {"username"})
})
public class PwjbUserInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
private String username;
private String password;
private String extra;
private Date gmtCreate;
private Date gmtModified;
}

43
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/ServerInfoDO.java Normal file
View File

@ -0,0 +1,43 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 服务器信息表（用于分配服务器唯一ID）
*
* @author tjq
* @since 2020/4/15
*/
@Data
@Entity
@NoArgsConstructor
@Table(
uniqueConstraints = {@UniqueConstraint(name = "uidx01_server_info", columnNames = "ip")},
indexes = {@Index(name = "idx01_server_info", columnList = "gmtModified")}
)
public class ServerInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* 服务器IP地址
*/
private String ip;
private Date gmtCreate;
private Date gmtModified;
public ServerInfoDO(String ip) {
this.ip = ip;
this.gmtCreate = new Date();
this.gmtModified = this.gmtCreate;
}
}

49
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/SundryDO.java Normal file
View File

@ -0,0 +1,49 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 杂项
* KKV 表存一些配置数据
*
* @author tjq
* @since 2024/2/15
*/
@Data
@Entity
@NoArgsConstructor
@Table(uniqueConstraints = {@UniqueConstraint(name = "uidx01_sundry", columnNames = {"pkey", "skey"})})
public class SundryDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* PKEY
*/
private String pkey;
/**
* SKEY
*/
private String skey;
/**
* 内容
*/
private String content;
/**
* 其他参数
*/
private String extra;
private Date gmtCreate;
private Date gmtModified;
}

82
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/UserInfoDO.java Normal file
View File

@ -0,0 +1,82 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 用户信息表
* PowerJob 自身维护的全部用户体系数据
* 5.0.0 可能不兼容改动：为了支持第三方登录，需要通过 username 与第三方登录系统做匹配，该列需要声明为唯一索引，确保全局唯一
*
* @author tjq
* @since 2020/4/12
*/
@Data
@Entity
@Table(uniqueConstraints = {
@UniqueConstraint(name = "uidx01_user_name", columnNames = {"username"})
},
indexes = {
@Index(name = "uidx02_user_info", columnList = "email")
})
public class UserInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* 账号类型
*/
private String accountType;
private String username;
/**
* since 5.0.0
* 昵称（第三方登陆的 username 很难识别，方便后续展示引入 nick）
*/
private String nick;
private String password;
/**
* 手机号
*/
private String phone;
/**
* 邮箱地址
*/
private String email;
/**
* webHook
*/
private String webHook;
/**
* JWT 登录的二次校验信息
*/
private String tokenLoginVerifyInfo;
/**
* 扩展字段 for 第三方
* PowerJob 内部不允许使用该字段
*/
private String extra;
/**
* 原始账号 username
*/
private String originUsername;
/**
* 账号当前状态
*/
private Integer status;
private Date gmtCreate;
private Date gmtModified;
}

53
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/UserRoleDO.java Normal file
View File

@ -0,0 +1,53 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.Data;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 用户角色表
*
* @author tjq
* @since 2023/3/20
*/
@Data
@Entity
@Table(indexes = {
@Index(name = "uidx01_user_id", columnList = "userId")
})
public class UserRoleDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* 授予角色的用户ID
*/
private Long userId;
/**
* 权限范围，namespace 还是 app
*/
private Integer scope;
/**
* 和 scope 一起组成授权目标，比如某个 app 或 某个 namespace
*/
private Long target;
/**
* 角色，比如 Observer
*/
private Integer role;
/**
* 扩展字段
*/
private String extra;
private Date gmtCreate;
private Date gmtModified;
}

82
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/WorkflowInfoDO.java Normal file
View File

@ -0,0 +1,82 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* DAG 工作流信息表
*
* @author tjq
* @since 2020/5/26
*/
@Data
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(indexes = {
@Index(name = "idx01_workflow_info",columnList = "appId,status,timeExpressionType,nextTriggerTime")
})
public class WorkflowInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
private String wfName;
private String wfDescription;
/**
* 所属应用ID
*/
private Long appId;
/**
* 工作流的DAG图信息（点线式DAG的json）
*/
@Lob
@Column
private String peDAG;
/* ************************** 定时参数 ************************** */
/**
* 时间表达式类型（CRON/API/FIX_RATE/FIX_DELAY）
*/
private Integer timeExpressionType;
/**
* 时间表达式，CRON/NULL/LONG/LONG
*/
private String timeExpression;
/**
* 最大同时运行的工作流个数，默认 1
*/
private Integer maxWfInstanceNum;
/**
* 1 正常运行，2 停止（不再调度）
*/
private Integer status;
/**
* 下一次调度时间
*/
private Long nextTriggerTime;
/**
* 工作流整体失败的报警
*/
private String notifyUserIds;
private Date gmtCreate;
private Date gmtModified;
private String extra;
private String lifecycle;
}

88
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/WorkflowInstanceInfoDO.java Normal file
View File

@ -0,0 +1,88 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.Date;
/**
* 工作流运行实例表
*
* @author tjq
* @since 2020/5/26
*/
@Data
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(
uniqueConstraints = {@UniqueConstraint(name = "uidx01_wf_instance", columnNames = {"wfInstanceId"})},
indexes = {
@Index(name = "idx01_wf_instance", columnList = "workflowId,status"),
@Index(name = "idx01_wf_instance", columnList = "appId,status,expectedTriggerTime")
}
)
public class WorkflowInstanceInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
/**
* 任务所属应用的ID，冗余提高查询效率
*/
private Long appId;
/**
* workflowInstanceId（任务实例表都使用单独的ID作为主键以支持潜在的分表需求）
*/
private Long wfInstanceId;
/**
* 上层工作流实例 ID （用于支持工作流嵌套）
*/
private Long parentWfInstanceId;
private Long workflowId;
/**
* workflow 状态（WorkflowInstanceStatus）
*/
private Integer status;
/**
* 工作流启动参数
*/
@Lob
@Column
private String wfInitParams;
/**
* 工作流上下文
*/
@Lob
@Column
private String wfContext;
@Lob
@Column
private String dag;
@Lob
@Column
private String result;
/**
* 预计触发时间
*/
private Long expectedTriggerTime;
/**
* 实际触发时间
*/
private Long actualTriggerTime;
/**
* 结束时间
*/
private Long finishedTime;
private Date gmtCreate;
private Date gmtModified;
}

82
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/WorkflowNodeInfoDO.java Normal file
View File

@ -0,0 +1,82 @@
package tech.powerjob.server.persistence.remote.model;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.hibernate.annotations.GenericGenerator;
import tech.powerjob.common.enums.WorkflowNodeType;
import javax.persistence.*;
import java.util.Date;
/**
* 工作流节点信息
* 记录了工作流中的任务节点个性化的配置信息
*
* @author Echo009
* @since 2021/1/23
*/
@Data
@Entity
@NoArgsConstructor
@AllArgsConstructor
@Table(indexes = {
@Index(name = "idx01_workflow_node_info", columnList = "workflowId,gmtCreate")
})
public class WorkflowNodeInfoDO {
@Id
@GeneratedValue(strategy = GenerationType.AUTO, generator = "native")
@GenericGenerator(name = "native", strategy = "native")
private Long id;
@Column(nullable = false)
private Long appId;
@Column
private Long workflowId;
/**
* 节点类型 {@link WorkflowNodeType}
*/
private Integer type;
/**
* 任务 ID
* 对于嵌套工作流类型的节点而言，这里存储是工作流 ID
*/
private Long jobId;
/**
* 节点名称，默认为对应的任务名称
*/
private String nodeName;
/**
* 节点参数
*/
@Lob
private String nodeParams;
/**
* 是否启用
*/
@Column(nullable = false)
private Boolean enable;
/**
* 是否允许失败跳过
*/
@Column(nullable = false)
private Boolean skipWhenFailed;
@Lob
private String extra;
/**
* 创建时间
*/
@Column(nullable = false)
private Date gmtCreate;
/**
* 更新时间
*/
@Column(nullable = false)
private Date gmtModified;
}

45
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/model/brief/BriefInstanceInfo.java Normal file
View File

@ -0,0 +1,45 @@
package tech.powerjob.server.persistence.remote.model.brief;
import lombok.Data;
/**
* @author Echo009
* @since 2022/9/13
*/
@Data
public class BriefInstanceInfo {
private Long appId;
private Long id;
/**
* 任务ID
*/
private Long jobId;
/**
* 任务所属应用的ID，冗余提高查询效率
*/
private Long instanceId;
/**
* 总共执行的次数（用于重试判断）
*/
private Long runningTimes;
public BriefInstanceInfo(Long appId, Long id, Long jobId, Long instanceId) {
this.appId = appId;
this.id = id;
this.jobId = jobId;
this.instanceId = instanceId;
}
public BriefInstanceInfo(Long appId, Long id, Long jobId, Long instanceId, Long runningTimes) {
this.appId = appId;
this.id = id;
this.jobId = jobId;
this.instanceId = instanceId;
this.runningTimes = runningTimes;
}
}

8
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/package-info.java Normal file
View File

@ -0,0 +1,8 @@
/**
* 关系型数据库持久层
* 需要注意实现平台无关性，比如禁止使用某些数据库特有的时间函数（MySQL:now()等等）
*
* @author tjq
* @since 2020/6/3
*/
package tech.powerjob.server.persistence.remote;

40
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/AppInfoRepository.java Normal file
View File

@ -0,0 +1,40 @@
package tech.powerjob.server.persistence.remote.repository;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import tech.powerjob.server.persistence.remote.model.AppInfoDO;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
/**
* AppInfo 数据访问层
*
* @author tjq
* @since 2020/4/1
*/
public interface AppInfoRepository extends JpaRepository<AppInfoDO, Long>, JpaSpecificationExecutor<AppInfoDO> {
Optional<AppInfoDO> findByAppName(String appName);
Page<AppInfoDO> findByAppNameLike(String condition, Pageable pageable);
/**
* 根据 currentServer 查询 appId
* 其实只需要 id，处于性能考虑可以直接写SQL只返回ID
*/
List<AppInfoDO> findAllByCurrentServer(String currentServer);
@Query(value = "select id from AppInfoDO where currentServer = :currentServer")
List<Long> listAppIdByCurrentServer(@Param("currentServer")String currentServer);
List<AppInfoDO> findAllByNamespaceId(Long namespaceId);
List<AppInfoDO> findAllByIdIn(Collection<Long> ids);
}

17
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/ContainerInfoRepository.java Normal file
View File

@ -0,0 +1,17 @@
package tech.powerjob.server.persistence.remote.repository;
import tech.powerjob.server.persistence.remote.model.ContainerInfoDO;
import org.springframework.data.jpa.repository.JpaRepository;
import java.util.List;
/**
* 容器信息 数据操作层
*
* @author tjq
* @since 2020/5/15
*/
public interface ContainerInfoRepository extends JpaRepository<ContainerInfoDO, Long> {
List<ContainerInfoDO> findByAppIdAndStatusNot(Long appId, Integer status);
}

147
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/InstanceInfoRepository.java Normal file
View File

@ -0,0 +1,147 @@
package tech.powerjob.server.persistence.remote.repository;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import tech.powerjob.server.persistence.remote.model.InstanceInfoDO;
import tech.powerjob.server.persistence.remote.model.brief.BriefInstanceInfo;
import javax.transaction.Transactional;
import java.util.Date;
import java.util.List;
/**
* JobLog 数据访问层
*
* @author tjq
* @since 2020/4/1
*/
public interface InstanceInfoRepository extends JpaRepository<InstanceInfoDO, Long>, JpaSpecificationExecutor<InstanceInfoDO> {
/**
* 统计当前JOB有多少实例正在运行
*/
@Query(value = "select count(*) from InstanceInfoDO where jobId = ?1 and status in ?2")
long countByJobIdAndStatusIn(long jobId, List<Integer> status);
List<InstanceInfoDO> findByJobIdAndStatusIn(long jobId, List<Integer> status);
/**
* 更新状态变更信息
*
* @param lastReportTime 最近一次上报时间
* @param modifyTime 更新时间
* @param runningTimes 运行次数
* @param instanceId 实例 ID
* @param status 目标状态
* @param oldStatus 旧状态
* @return 更新记录数
*/
@Transactional(rollbackOn = Exception.class)
@Modifying
@Query(value = "update InstanceInfoDO set lastReportTime = :lastReportTime, gmtModified = :modifyTime, runningTimes = :runningTimes, status = :status where instanceId = :instanceId and status = :oldStatus")
int updateStatusChangeInfoByInstanceIdAndStatus(@Param("lastReportTime") long lastReportTime, @Param("modifyTime") Date modifyTime, @Param("runningTimes") long runningTimes, @Param("status") int status, @Param("instanceId") long instanceId, @Param("oldStatus") int oldStatus);
/**
* 更新任务执行记录内容（DispatchService专用）
*
* @param instanceId 实例 ID
* @param status 状态
* @param actualTriggerTime 实际调度时间
* @param finishedTime 完成时间
* @param taskTrackerAddress taskTracker 地址
* @param result 结果
* @param modifyTime 更新时间
* @return 更新记录数量
*/
@Transactional(rollbackOn = Exception.class)
@Modifying
@CanIgnoreReturnValue
@Query(value = "update InstanceInfoDO set status = :status, actualTriggerTime = :actualTriggerTime, finishedTime = :finishedTime, taskTrackerAddress = :taskTrackerAddress, result = :result, gmtModified = :modifyTime where instanceId = :instanceId")
int update4TriggerFailed(@Param("instanceId") long instanceId, @Param("status") int status, @Param("actualTriggerTime") long actualTriggerTime, @Param("finishedTime") long finishedTime, @Param("taskTrackerAddress") String taskTrackerAddress, @Param("result") String result, @Param("modifyTime") Date modifyTime);
/**
* 更新任务执行记录内容（DispatchService专用）
*
* @param instanceId 任务实例ID，分布式唯一
* @param status 状态
* @param actualTriggerTime 实际调度时间
* @param taskTrackerAddress taskTracker 地址
* @param modifyTime 更新时间
* @param oldStatus 旧状态
* @return 更新记录数量
*/
@Transactional(rollbackOn = Exception.class)
@Modifying
@CanIgnoreReturnValue
@Query(value = "update InstanceInfoDO set status = :status, actualTriggerTime = :actualTriggerTime, taskTrackerAddress = :taskTrackerAddress, gmtModified = :modifyTime where instanceId = :instanceId and status = :oldStatus")
int update4TriggerSucceed(@Param("instanceId") long instanceId, @Param("status") int status, @Param("actualTriggerTime") long actualTriggerTime, @Param("taskTrackerAddress") String taskTrackerAddress, @Param("modifyTime") Date modifyTime, @Param("oldStatus") int oldStatus);
@Transactional(rollbackOn = Exception.class)
@Modifying
@CanIgnoreReturnValue
@Query(value = "update InstanceInfoDO set status = :status, gmtModified = :modifyTime where instanceId = :instanceId and status = :originStatus ")
int updateStatusAndGmtModifiedByInstanceIdAndOriginStatus(@Param("instanceId") long instanceId, @Param("originStatus") int originStatus, @Param("status") int status, @Param("modifyTime") Date modifyTime);
@Transactional(rollbackOn = Exception.class)
@Modifying
@CanIgnoreReturnValue
@Query(value = "update InstanceInfoDO set status = :status, gmtModified = :modifyTime where instanceId in (:instanceIdList) and status = :originStatus ")
int updateStatusAndGmtModifiedByInstanceIdListAndOriginStatus(@Param("instanceIdList") List<Long> instanceIdList, @Param("originStatus") int originStatus, @Param("status") int status, @Param("modifyTime") Date modifyTime);
/**
* 更新固定频率任务的执行记录
*
* @param instanceId 任务实例ID，分布式唯一
* @param status 状态
* @param runningTimes 执行次数
* @param modifyTime 更新时间
* @return 更新记录数量
*/
@Modifying
@Transactional(rollbackOn = Exception.class)
@CanIgnoreReturnValue
@Query(value = "update InstanceInfoDO set status = :status, runningTimes = :runningTimes, gmtModified = :modifyTime where instanceId = :instanceId")
int update4FrequentJob(@Param("instanceId") long instanceId, @Param("status") int status, @Param("runningTimes") long runningTimes, @Param("modifyTime") Date modifyTime);
List<InstanceInfoDO> findAllByAppIdInAndStatusAndExpectedTriggerTimeLessThan(@Param("appIds") List<Long> appIds, @Param("status") int status, @Param("time") long time, Pageable pageable);
@Query(value = "select new tech.powerjob.server.persistence.remote.model.brief.BriefInstanceInfo(i.appId,i.id,i.jobId,i.instanceId) from InstanceInfoDO i where i.appId in (:appIds) and i.status = :status and i.actualTriggerTime < :time")
List<BriefInstanceInfo> selectBriefInfoByAppIdInAndStatusAndActualTriggerTimeLessThan(@Param("appIds") List<Long> appIds, @Param("status") int status, @Param("time") long time, Pageable pageable);
@Query(value = "select new tech.powerjob.server.persistence.remote.model.brief.BriefInstanceInfo(i.appId,i.id,i.jobId,i.instanceId,i.runningTimes) from InstanceInfoDO i where i.appId in (:appIds) and i.status = :status and i.gmtModified < :time")
List<BriefInstanceInfo> selectBriefInfoByAppIdInAndStatusAndGmtModifiedBefore(@Param("appIds") List<Long> appIds, @Param("status") int status, @Param("time") Date time, Pageable pageable);
InstanceInfoDO findByInstanceId(long instanceId);
/* --数据统计-- */
@Query(value = "select count(*) from InstanceInfoDO where appId = ?1 and status = ?2")
long countByAppIdAndStatus(long appId, int status);
long countByAppIdAndStatusAndGmtCreateAfter(long appId, int status, Date time);
@Query(value = "select distinct jobId from InstanceInfoDO where jobId in ?1 and status in ?2")
List<Long> findByJobIdInAndStatusIn(List<Long> jobIds, List<Integer> status);
/**
* 删除历史数据，JPA自带的删除居然是根据ID循环删，2000条数据删了几秒，也太拉垮了吧...
* 结果只能用 int 接收
*
* @param time 更新时间阈值
* @param status 状态
* @return 删除记录数
*/
@Modifying
@Transactional(rollbackOn = Exception.class)
@Query(value = "delete from InstanceInfoDO where gmtModified < ?1 and status in ?2")
int deleteAllByGmtModifiedBeforeAndStatusIn(Date time, List<Integer> status);
}

50
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/JobInfoRepository.java Normal file
View File

@ -0,0 +1,50 @@
package tech.powerjob.server.persistence.remote.repository;
import tech.powerjob.server.persistence.remote.model.JobInfoDO;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Query;
import java.util.Collection;
import java.util.List;
import java.util.Set;
/**
* JobInfo 数据访问层
*
* @author tjq
* @since 2020/4/1
*/
public interface JobInfoRepository extends JpaRepository<JobInfoDO, Long>, JpaSpecificationExecutor<JobInfoDO> {
/**
* 调度专用
*/
List<JobInfoDO> findByAppIdInAndStatusAndTimeExpressionTypeAndNextTriggerTimeLessThanEqual(List<Long> appIds, int status, int timeExpressionType, long time);
@Query(value = "select id from JobInfoDO where appId in ?1 and status = ?2 and timeExpressionType in ?3")
List<Long> findByAppIdInAndStatusAndTimeExpressionTypeIn(List<Long> appIds, int status, List<Integer> timeTypes);
Page<JobInfoDO> findByAppIdAndStatusNot(Long appId, int status, Pageable pageable);
Page<JobInfoDO> findByAppIdAndJobNameLikeAndStatusNot(Long appId, String condition, int status, Pageable pageable);
/**
* 校验工作流包含的任务
* @param appId APP ID
* @param statusSet 状态列表
* @param jobIds 任务ID
* @return 数量
*/
long countByAppIdAndStatusInAndIdIn(Long appId, Set<Integer> statusSet , Set<Long> jobIds);
long countByAppIdAndStatusNot(long appId, int status);
List<JobInfoDO> findByAppId(Long appId);
List<JobInfoDO> findByIdIn(Collection<Long> jobIds);
}

22
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/NamespaceRepository.java Normal file
View File

@ -0,0 +1,22 @@
package tech.powerjob.server.persistence.remote.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import tech.powerjob.server.persistence.remote.model.NamespaceDO;
import java.util.Collection;
import java.util.List;
import java.util.Optional;
/**
* 命名空间
*
* @author tjq
* @since 2023/9/3
*/
public interface NamespaceRepository extends JpaRepository<NamespaceDO, Long>, JpaSpecificationExecutor<NamespaceDO> {
Optional<NamespaceDO> findByCode(String code);
List<NamespaceDO> findAllByIdIn(Collection<Long> ids);
}

28
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/OmsLockRepository.java Normal file
View File

@ -0,0 +1,28 @@
package tech.powerjob.server.persistence.remote.repository;
import tech.powerjob.server.persistence.remote.model.OmsLockDO;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import javax.transaction.Transactional;
/**
* 利用唯一性约束作为数据库锁
*
* @author tjq
* @since 2020/4/2
*/
public interface OmsLockRepository extends JpaRepository<OmsLockDO, Long> {
@Modifying
@Transactional(rollbackOn = Exception.class)
@Query(value = "delete from OmsLockDO where lockName = ?1")
int deleteByLockName(String lockName);
OmsLockDO findByLockName(String lockName);
@Modifying
@Transactional(rollbackOn = Exception.class)
int deleteByOwnerIP(String ip);
}

17
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/PwjbUserInfoRepository.java Normal file
View File

@ -0,0 +1,17 @@
package tech.powerjob.server.persistence.remote.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import tech.powerjob.server.persistence.remote.model.PwjbUserInfoDO;
import java.util.Optional;
/**
* PwjbUserInfoRepository
*
* @author tjq
* @since 2024/2/13
*/
public interface PwjbUserInfoRepository extends JpaRepository<PwjbUserInfoDO, Long> {
Optional<PwjbUserInfoDO> findByUsername(String username);
}

39
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/ServerInfoRepository.java Normal file
View File

@ -0,0 +1,39 @@
package tech.powerjob.server.persistence.remote.repository;
import tech.powerjob.server.persistence.remote.model.ServerInfoDO;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import javax.transaction.Transactional;
import java.util.Date;
/**
* 服务器信息 数据操作层
*
* @author tjq
* @since 2020/4/15
*/
public interface ServerInfoRepository extends JpaRepository<ServerInfoDO, Long> {
ServerInfoDO findByIp(String ip);
@Transactional(rollbackOn = Exception.class)
@Modifying
@CanIgnoreReturnValue
@Query(value = "update ServerInfoDO set gmtModified = :gmtModified where ip = :ip")
int updateGmtModifiedByIp(@Param("ip") String ip, @Param("gmtModified") Date gmtModified);
@Transactional(rollbackOn = Exception.class)
@Modifying
@CanIgnoreReturnValue
@Query(value = "update ServerInfoDO set id = :id where ip = :ip")
int updateIdByIp(@Param("id") long id, @Param("ip") String ip);
@Transactional(rollbackOn = Exception.class)
@Modifying
@Query(value = "delete from ServerInfoDO where gmtModified < ?1")
int deleteByGmtModifiedBefore(Date threshold);
}

20
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/SundryRepository.java Normal file
View File

@ -0,0 +1,20 @@
package tech.powerjob.server.persistence.remote.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import tech.powerjob.server.persistence.remote.model.SundryDO;
import java.util.List;
import java.util.Optional;
/**
* SundryRepository
*
* @author tjq
* @since 2024/2/15
*/
public interface SundryRepository extends JpaRepository<SundryDO, Long> {
List<SundryDO> findAllByPkey(String pkey);
Optional<SundryDO> findByPkeyAndSkey(String pkey, String skey);
}

23
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/UserInfoRepository.java Normal file
View File

@ -0,0 +1,23 @@
package tech.powerjob.server.persistence.remote.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import tech.powerjob.server.persistence.remote.model.UserInfoDO;
import java.util.List;
import java.util.Optional;
/**
* 用户信息表数据库访问层
*
* @author tjq
* @since 2020/4/12
*/
public interface UserInfoRepository extends JpaRepository<UserInfoDO, Long>, JpaSpecificationExecutor<UserInfoDO> {
Optional<UserInfoDO> findByUsername(String username);
List<UserInfoDO> findByUsernameLike(String username);
List<UserInfoDO> findByIdIn(List<Long> userIds);
}

23
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/UserRoleRepository.java Normal file
View File

@ -0,0 +1,23 @@
package tech.powerjob.server.persistence.remote.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import tech.powerjob.server.persistence.remote.model.UserRoleDO;
import java.util.List;
/**
* UserRoleRepository
*
* @author tjq
* @since 2023/3/20
*/
public interface UserRoleRepository extends JpaRepository<UserRoleDO, Long> {
List<UserRoleDO> findAllByUserId(Long userId);
List<UserRoleDO> findAllByScopeAndTarget(Integer scope, Long target);
List<UserRoleDO> findAllByScopeAndTargetAndRoleAndUserId(Integer scope, Long target, Integer role, Long userId);
List<UserRoleDO> findAllByUserIdAndScope(Long userId, Integer scope);
}

37
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/WorkflowInfoRepository.java Normal file
View File

@ -0,0 +1,37 @@
package tech.powerjob.server.persistence.remote.repository;
import tech.powerjob.server.persistence.remote.model.WorkflowInfoDO;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
import java.util.List;
/**
* DAG 工作流 数据操作层
*
* @author tjq
* @since 2020/5/26
*/
public interface WorkflowInfoRepository extends JpaRepository<WorkflowInfoDO, Long> {
List<WorkflowInfoDO> findByAppIdInAndStatusAndTimeExpressionTypeAndNextTriggerTimeLessThanEqual(List<Long> appIds, int status, int timeExpressionType, long time);
/**
* 查询指定 APP 下所有的工作流信息
*
* @param appId APP ID
* @return 该 APP 下的所有工作流信息
*/
List<WorkflowInfoDO> findByAppId(Long appId);
/**
* 对外查询（list）三兄弟
*/
Page<WorkflowInfoDO> findByAppIdAndStatusNot(Long appId, int nStatus, Pageable pageable);
Page<WorkflowInfoDO> findByIdAndStatusNot(Long id, int nStatus, Pageable pageable);
Page<WorkflowInfoDO> findByAppIdAndStatusNotAndWfNameLike(Long appId, int nStatus, String condition, Pageable pageable);
}

56
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/WorkflowInstanceInfoRepository.java Normal file
View File

@ -0,0 +1,56 @@
package tech.powerjob.server.persistence.remote.repository;
import tech.powerjob.server.persistence.remote.model.WorkflowInstanceInfoDO;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import javax.transaction.Transactional;
import java.util.Date;
import java.util.List;
import java.util.Optional;
/**
* 工作流运行实例数据操作
*
* @author tjq
* @since 2020/5/26
*/
public interface WorkflowInstanceInfoRepository extends JpaRepository<WorkflowInstanceInfoDO, Long> {
/**
* 查找对应工作流实例
* @param wfInstanceId 实例 ID
* @return 工作流实例
*/
Optional<WorkflowInstanceInfoDO> findByWfInstanceId(Long wfInstanceId);
/**
* 删除历史数据，JPA自带的删除居然是根据ID循环删，2000条数据删了几秒，也太拉垮了吧...
* 结果只能用 int 接收
* @param time 更新时间阈值
* @param status 状态列表
* @return 删除的记录条数
*/
@Modifying
@Transactional(rollbackOn = Exception.class)
@Query(value = "delete from WorkflowInstanceInfoDO where gmtModified < ?1 and status in ?2")
int deleteAllByGmtModifiedBeforeAndStatusIn(Date time, List<Integer> status);
/**
* 统计该工作流下处于对应状态的实例数量
* @param workflowId 工作流 ID
* @param status 状态列表
* @return 更新的记录条数
*/
int countByWorkflowIdAndStatusIn(Long workflowId, List<Integer> status);
/**
* 加载期望调度时间小于给定阈值的
* @param appIds 应用 ID 列表
* @param status 状态
* @param time 期望调度时间阈值
* @return 工作流实例列表
*/
List<WorkflowInstanceInfoDO> findByAppIdInAndStatusAndExpectedTriggerTimeLessThan(List<Long> appIds, int status, long time);
}

49
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/remote/repository/WorkflowNodeInfoRepository.java Normal file
View File

@ -0,0 +1,49 @@
package tech.powerjob.server.persistence.remote.repository;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import tech.powerjob.server.persistence.remote.model.WorkflowNodeInfoDO;
import javax.transaction.Transactional;
import java.util.Date;
import java.util.List;
/**
* WorkflowNodeInfo 数据访问层
*
* @author Echo009
* @since 2021/2/1
*/
public interface WorkflowNodeInfoRepository extends JpaRepository<WorkflowNodeInfoDO, Long> {
/**
* 根据工作流id查找所有的节点
*
* @param workflowId 工作流id
* @return 节点信息集合
*/
List<WorkflowNodeInfoDO> findByWorkflowId(Long workflowId);
/**
* 根据工作流节点 ID 删除节点
*
* @param workflowId 工作流ID
* @param id 节点 ID
* @return 删除记录数
*/
int deleteByWorkflowIdAndIdNotIn(Long workflowId, List<Long> id);
/**
* 删除工作流 ID 为空，且创建时间早于指定时间的节点信息
*
* @param crtTimeThreshold 创建时间阈值
* @return 删除记录条数
*/
@Modifying
@Transactional(rollbackOn = Exception.class)
@Query(value = "delete from WorkflowNodeInfoDO where workflowId is null and gmtCreate < ?1")
int deleteAllByWorkflowIdIsNullAndGmtCreateBefore(Date crtTimeThreshold);
}

49
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/AbstractDFsService.java Normal file
View File

@ -0,0 +1,49 @@
package tech.powerjob.server.persistence.storage;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.core.env.Environment;
import tech.powerjob.server.common.aware.ServerInfoAware;
import tech.powerjob.server.common.module.ServerInfo;
import tech.powerjob.server.extension.dfs.DFsService;
/**
* AbstractDFsService
*
* @author tjq
* @since 2023/7/28
*/
@Slf4j
public abstract class AbstractDFsService implements DFsService, ApplicationContextAware, ServerInfoAware, DisposableBean {
protected ServerInfo serverInfo;
protected ApplicationContext applicationContext;
public AbstractDFsService() {
log.info("[DFsService] invoke [{}]'s constructor", this.getClass().getName());
}
abstract protected void init(ApplicationContext applicationContext);
protected static final String PROPERTY_KEY = "oms.storage.dfs";
protected static String fetchProperty(Environment environment, String dfsType, String key) {
String pKey = String.format("%s.%s.%s", PROPERTY_KEY, dfsType, key);
return environment.getProperty(pKey);
}
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
this.applicationContext = applicationContext;
log.info("[DFsService] invoke [{}]'s setApplicationContext", this.getClass().getName());
init(applicationContext);
}
@Override
public void setServerInfo(ServerInfo serverInfo) {
this.serverInfo = serverInfo;
}
}

15
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/Constants.java Normal file
View File

@ -0,0 +1,15 @@
package tech.powerjob.server.persistence.storage;
/**
* Constants
*
* @author tjq
* @since 2023/7/30
*/
public class Constants {
public static final String LOG_BUCKET = "log";
public static final String CONTAINER_BUCKET = "container";
}

53
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/StorageConfiguration.java Normal file
View File

@ -0,0 +1,53 @@
package tech.powerjob.server.persistence.storage;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Conditional;
import org.springframework.context.annotation.Configuration;
import tech.powerjob.server.extension.dfs.DFsService;
import tech.powerjob.server.persistence.storage.impl.*;
/**
* 初始化内置的存储服务
*
* @author tjq
* @since 2023/7/30
*/
@Configuration
public class StorageConfiguration {
@Bean
@Conditional(GridFsService.GridFsCondition.class)
public DFsService initGridFs() {
return new GridFsService();
}
@Bean
@Conditional(MySqlSeriesDfsService.MySqlSeriesCondition.class)
public DFsService initDbFs() {
return new MySqlSeriesDfsService();
}
@Bean
@Conditional(PostgresqlSeriesDfsService.PostgresqlSeriesCondition.class)
public DFsService initPGDbFs() {
return new PostgresqlSeriesDfsService();
}
@Bean
@Conditional(AliOssService.AliOssCondition.class)
public DFsService initAliOssFs() {
return new AliOssService();
}
@Bean
@Conditional(MinioOssService.MinioOssCondition.class)
public DFsService initMinioOssFs() {
return new MinioOssService();
}
@Bean
@Conditional(EmptyDFsService.EmptyCondition.class)
public DFsService initEmptyDfs() {
return new EmptyDFsService();
}
}

229
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/AliOssService.java Normal file
View File

@ -0,0 +1,229 @@
package tech.powerjob.server.persistence.storage.impl;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSClientBuilder;
import com.aliyun.oss.OSSException;
import com.aliyun.oss.common.auth.CredentialsProvider;
import com.aliyun.oss.common.auth.CredentialsProviderFactory;
import com.aliyun.oss.common.auth.DefaultCredentialProvider;
import com.aliyun.oss.model.DownloadFileRequest;
import com.aliyun.oss.model.ObjectMetadata;
import com.aliyun.oss.model.PutObjectRequest;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Conditional;
import org.springframework.core.env.Environment;
import tech.powerjob.server.extension.dfs.*;
import tech.powerjob.server.persistence.storage.AbstractDFsService;
import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition;
import javax.annotation.Priority;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* Alibaba OSS support
* <a href="https://www.aliyun.com/product/oss">海量、安全、低成本、高可靠的云存储服务</a>
* 配置项：
* oms.storage.dfs.alioss.endpoint
* oms.storage.dfs.alioss.bucket
* oms.storage.dfs.alioss.credential_type
* oms.storage.dfs.alioss.ak
* oms.storage.dfs.alioss.sk
* oms.storage.dfs.alioss.token
*
* @author tjq
* @since 2023/7/30
*/
@Slf4j
@Priority(value = Integer.MAX_VALUE - 1)
@Conditional(AliOssService.AliOssCondition.class)
public class AliOssService extends AbstractDFsService {
private static final String TYPE_ALI_OSS = "alioss";
private static final String KEY_ENDPOINT = "endpoint";
private static final String KEY_BUCKET = "bucket";
private static final String KEY_CREDENTIAL_TYPE = "credential_type";
private static final String KEY_AK = "ak";
private static final String KEY_SK = "sk";
private static final String KEY_TOKEN = "token";
private OSS oss;
private String bucket;
private static final int DOWNLOAD_PART_SIZE = 10240;
private static final String NO_SUCH_KEY = "NoSuchKey";
@Override
public void store(StoreRequest storeRequest) throws IOException {
ObjectMetadata objectMetadata = new ObjectMetadata();
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, parseFileName(storeRequest.getFileLocation()), storeRequest.getLocalFile(), objectMetadata);
oss.putObject(putObjectRequest);
}
@Override
public void download(DownloadRequest downloadRequest) throws IOException {
FileLocation dfl = downloadRequest.getFileLocation();
DownloadFileRequest downloadFileRequest = new DownloadFileRequest(bucket, parseFileName(dfl), downloadRequest.getTarget().getAbsolutePath(), DOWNLOAD_PART_SIZE);
try {
FileUtils.forceMkdirParent(downloadRequest.getTarget());
oss.downloadFile(downloadFileRequest);
} catch (Throwable t) {
ExceptionUtils.rethrow(t);
}
}
@Override
public Optional<FileMeta> fetchFileMeta(FileLocation fileLocation) throws IOException {
try {
ObjectMetadata objectMetadata = oss.getObjectMetadata(bucket, parseFileName(fileLocation));
return Optional.ofNullable(objectMetadata).map(ossM -> {
Map<String, Object> metaInfo = Maps.newHashMap();
metaInfo.putAll(ossM.getRawMetadata());
if (ossM.getUserMetadata() != null) {
metaInfo.putAll(ossM.getUserMetadata());
}
return new FileMeta()
.setLastModifiedTime(ossM.getLastModified())
.setLength(ossM.getContentLength())
.setMetaInfo(metaInfo);
});
} catch (OSSException oe) {
String errorCode = oe.getErrorCode();
if (NO_SUCH_KEY.equalsIgnoreCase(errorCode)) {
return Optional.empty();
}
ExceptionUtils.rethrow(oe);
}
return Optional.empty();
}
private static String parseFileName(FileLocation fileLocation) {
return String.format("%s/%s", fileLocation.getBucket(), fileLocation.getName());
}
void initOssClient(String endpoint, String bucket, String mode, String ak, String sk, String token) throws Exception {
log.info("[AliOssService] init OSS by config: endpoint={},bucket={},credentialType={},ak={},sk={},token={}", endpoint, bucket, mode, ak, sk, token);
if (StringUtils.isEmpty(bucket)) {
throw new IllegalArgumentException("'oms.storage.dfs.alioss.bucket' can't be empty, please creat a bucket in aliyun oss console then config it to powerjob");
}
this.bucket = bucket;
CredentialsProvider credentialsProvider;
CredentialType credentialType = CredentialType.parse(mode);
switch (credentialType) {
case PWD:
credentialsProvider = new DefaultCredentialProvider(ak, sk, token);
break;
case SYSTEM_PROPERTY:
credentialsProvider = CredentialsProviderFactory.newSystemPropertiesCredentialsProvider();
break;
default:
credentialsProvider = CredentialsProviderFactory.newEnvironmentVariableCredentialsProvider();
}
this.oss = new OSSClientBuilder().build(endpoint, credentialsProvider);
log.info("[AliOssService] initialize successfully, THIS_WILL_BE_THE_STORAGE_LAYER.");
}
@Override
public void cleanExpiredFiles(String bucket, int days) {
/*
阿里云 OSS 自带生命周期管理，请参考文档进行配置，代码层面不进行实现（浪费服务器资源）https://help.aliyun.com/zh/oss/user-guide/overview-54
阿里云 OSS 自带生命周期管理，请参考文档进行配置，代码层面不进行实现（浪费服务器资源）https://help.aliyun.com/zh/oss/user-guide/overview-54
阿里云 OSS 自带生命周期管理，请参考文档进行配置，代码层面不进行实现（浪费服务器资源）https://help.aliyun.com/zh/oss/user-guide/overview-54
*/
}
@Override
public void destroy() throws Exception {
oss.shutdown();
}
@Override
protected void init(ApplicationContext applicationContext) {
Environment environment = applicationContext.getEnvironment();
String endpoint = fetchProperty(environment, TYPE_ALI_OSS, KEY_ENDPOINT);
String bkt = fetchProperty(environment, TYPE_ALI_OSS, KEY_BUCKET);
String ct = fetchProperty(environment, TYPE_ALI_OSS, KEY_CREDENTIAL_TYPE);
String ak = fetchProperty(environment, TYPE_ALI_OSS, KEY_AK);
String sk = fetchProperty(environment, TYPE_ALI_OSS, KEY_SK);
String token = fetchProperty(environment, TYPE_ALI_OSS, KEY_TOKEN);
try {
initOssClient(endpoint, bkt, ct, ak, sk, token);
} catch (Exception e) {
ExceptionUtils.rethrow(e);
}
}
@Getter
@AllArgsConstructor
enum CredentialType {
/**
* 从环境读取
*/
ENV("env"),
/**
* 系统配置
*/
SYSTEM_PROPERTY("sys"),
/**
* 从账号密码读取
*/
PWD("pwd")
;
private final String code;
/**
* parse credential type
* @param mode oms.storage.dfs.alioss.credential_type
* @return CredentialType
*/
public static CredentialType parse(String mode) {
for (CredentialType credentialType : values()) {
if (StringUtils.equalsIgnoreCase(credentialType.code, mode)) {
return credentialType;
}
}
return PWD;
}
}
public static class AliOssCondition extends PropertyAndOneBeanCondition {
@Override
protected List<String> anyConfigKey() {
return Lists.newArrayList("oms.storage.dfs.alioss.endpoint");
}
@Override
protected Class<?> beanType() {
return DFsService.class;
}
}
}

61
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/EmptyDFsService.java Normal file
View File

@ -0,0 +1,61 @@
package tech.powerjob.server.persistence.storage.impl;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Conditional;
import tech.powerjob.server.extension.dfs.*;
import tech.powerjob.server.persistence.storage.AbstractDFsService;
import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition;
import javax.annotation.Priority;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
/**
* EmptyDFsService
*
* @author tjq
* @since 2023/7/30
*/
@Slf4j
@Priority(value = Integer.MAX_VALUE)
@Conditional(EmptyDFsService.EmptyCondition.class)
public class EmptyDFsService extends AbstractDFsService {
@Override
public void store(StoreRequest storeRequest) throws IOException {
}
@Override
public void download(DownloadRequest downloadRequest) throws IOException {
}
@Override
public Optional<FileMeta> fetchFileMeta(FileLocation fileLocation) throws IOException {
return Optional.empty();
}
@Override
public void destroy() throws Exception {
}
@Override
protected void init(ApplicationContext applicationContext) {
log.info("[EmptyDFsService] initialize successfully, THIS_WILL_BE_THE_STORAGE_LAYER.");
}
public static class EmptyCondition extends PropertyAndOneBeanCondition {
@Override
protected List<String> anyConfigKey() {
return null;
}
@Override
protected Class<?> beanType() {
return DFsService.class;
}
}
}

176
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/GridFsService.java Normal file
View File

@ -0,0 +1,176 @@
package tech.powerjob.server.persistence.storage.impl;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.mongodb.ConnectionString;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.gridfs.GridFSBucket;
import com.mongodb.client.gridfs.GridFSBuckets;
import com.mongodb.client.gridfs.GridFSDownloadStream;
import com.mongodb.client.gridfs.GridFSFindIterable;
import com.mongodb.client.gridfs.model.GridFSFile;
import com.mongodb.client.model.Filters;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.bson.conversions.Bson;
import org.bson.types.ObjectId;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Conditional;
import org.springframework.core.env.Environment;
import tech.powerjob.server.extension.dfs.*;
import tech.powerjob.server.persistence.storage.AbstractDFsService;
import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition;
import javax.annotation.Priority;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* 使用 MongoDB GridFS 作为底层存储
* 配置用法：oms.storage.dfs.mongodb.uri=mongodb+srv://zqq:No1Bug2Please3!@cluster0.wie54.gcp.mongodb.net/powerjob_daily?retryWrites=true&w=majority
*
* @author tjq
* @since 2023/7/28
*/
@Slf4j
@Priority(value = Integer.MAX_VALUE - 10)
@Conditional(GridFsService.GridFsCondition.class)
public class GridFsService extends AbstractDFsService {
private MongoClient mongoClient;
private MongoDatabase db;
private final Map<String, GridFSBucket> bucketCache = Maps.newConcurrentMap();
private static final String TYPE_MONGO = "mongodb";
private static final String KEY_URI = "uri";
private static final String SPRING_MONGO_DB_CONFIG_KEY = "spring.data.mongodb.uri";
@Override
public void store(StoreRequest storeRequest) throws IOException {
GridFSBucket bucket = getBucket(storeRequest.getFileLocation().getBucket());
try (BufferedInputStream bis = new BufferedInputStream(Files.newInputStream(storeRequest.getLocalFile().toPath()))) {
bucket.uploadFromStream(storeRequest.getFileLocation().getName(), bis);
}
}
@Override
public void download(DownloadRequest downloadRequest) throws IOException {
GridFSBucket bucket = getBucket(downloadRequest.getFileLocation().getBucket());
FileUtils.forceMkdirParent(downloadRequest.getTarget());
try (GridFSDownloadStream gis = bucket.openDownloadStream(downloadRequest.getFileLocation().getName());
BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(downloadRequest.getTarget().toPath()))
) {
byte[] buffer = new byte[1024];
int bytes = 0;
while ((bytes = gis.read(buffer)) != -1) {
bos.write(buffer, 0, bytes);
}
bos.flush();
}
}
@Override
public Optional<FileMeta> fetchFileMeta(FileLocation fileLocation) throws IOException {
GridFSBucket bucket = getBucket(fileLocation.getBucket());
GridFSFindIterable files = bucket.find(Filters.eq("filename", fileLocation.getName()));
GridFSFile first = files.first();
if (first == null) {
return Optional.empty();
}
return Optional.of(new FileMeta()
.setLength(first.getLength())
.setLastModifiedTime(first.getUploadDate())
.setMetaInfo(first.getMetadata()));
}
@Override
public void cleanExpiredFiles(String bucketName, int days) {
Stopwatch sw = Stopwatch.createStarted();
Date date = DateUtils.addDays(new Date(), -days);
GridFSBucket bucket = getBucket(bucketName);
Bson filter = Filters.lt("uploadDate", date);
// 循环删除性能很差？我猜你肯定没看过官方实现[狗头]：org.springframework.data.mongodb.gridfs.GridFsTemplate.delete
bucket.find(filter).forEach(gridFSFile -> {
ObjectId objectId = gridFSFile.getObjectId();
try {
bucket.delete(objectId);
log.info("[GridFsService] deleted {}#{}", bucketName, objectId);
}catch (Exception e) {
log.error("[GridFsService] deleted {}#{} failed.", bucketName, objectId, e);
}
});
log.info("[GridFsService] clean bucket({}) successfully, delete all files before {}, using {}.", bucketName, date, sw.stop());
}
private GridFSBucket getBucket(String bucketName) {
return bucketCache.computeIfAbsent(bucketName, ignore -> GridFSBuckets.create(db, bucketName));
}
private String parseMongoUri(Environment environment) {
// 优先从新的规则读取
String uri = fetchProperty(environment, TYPE_MONGO, KEY_URI);
if (StringUtils.isNotEmpty(uri)) {
return uri;
}
// 兼容 4.3.3 前的逻辑，读取 SpringMongoDB 配置
return environment.getProperty(SPRING_MONGO_DB_CONFIG_KEY);
}
void initMongo(String uri) {
log.info("[GridFsService] mongoDB uri: {}", uri);
if (StringUtils.isEmpty(uri)) {
log.warn("[GridFsService] uri is empty, GridFsService is off now!");
return;
}
ConnectionString connectionString = new ConnectionString(uri);
mongoClient = MongoClients.create(connectionString);
if (StringUtils.isEmpty(connectionString.getDatabase())) {
log.warn("[GridFsService] can't find database info from uri, will use [powerjob] as default, please make sure you have created the database 'powerjob'");
}
db = mongoClient.getDatabase(Optional.ofNullable(connectionString.getDatabase()).orElse("powerjob"));
log.info("[GridFsService] initialize MongoDB and GridFS successfully, will use mongodb GridFs as storage layer.");
}
@Override
public void destroy() throws Exception {
mongoClient.close();
}
@Override
protected void init(ApplicationContext applicationContext) {
String uri = parseMongoUri(applicationContext.getEnvironment());
initMongo(uri);
log.info("[GridFsService] initialize successfully, THIS_WILL_BE_THE_STORAGE_LAYER.");
}
public static class GridFsCondition extends PropertyAndOneBeanCondition {
@Override
protected List<String> anyConfigKey() {
return Lists.newArrayList("spring.data.mongodb.uri", "oms.storage.dfs.mongodb.uri");
}
@Override
protected Class<?> beanType() {
return DFsService.class;
}
}
}

258
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MinioOssService.java Normal file
View File

@ -0,0 +1,258 @@
package tech.powerjob.server.persistence.storage.impl;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.*;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Conditional;
import org.springframework.core.env.Environment;
import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition;
import tech.powerjob.server.extension.dfs.*;
import tech.powerjob.server.persistence.storage.AbstractDFsService;
import javax.annotation.Priority;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* MINIO support
* <a href="https://min.io/">High Performance Object Storage</a>
* 配置项：
* oms.storage.dfs.minio.endpoint
* oms.storage.dfs.minio.bucketName
* oms.storage.dfs.minio.accessKey
* oms.storage.dfs.minio.secretKey
*
* @author xinyi
* @since 2023/8/21
*/
@Slf4j
@Priority(value = Integer.MAX_VALUE - 3)
@Conditional(MinioOssService.MinioOssCondition.class)
public class MinioOssService extends AbstractDFsService {
private static final String TYPE_MINIO = "minio";
private static final String KEY_ENDPOINT = "endpoint";
private static final String KEY_BUCKET_NAME = "bucketName";
private static final String ACCESS_KEY = "accessKey";
private static final String SECRET_KEY = "secretKey";
private AmazonS3 amazonS3;
private String bucket;
private static final String NOT_FOUNT = "404 Not Found";
@Override
public void store(StoreRequest storeRequest) {
try {
String fileName = parseFileName(storeRequest.getFileLocation());
// 创建 PutObjectRequest 对象
PutObjectRequest request = new PutObjectRequest(this.bucket, fileName, storeRequest.getLocalFile());
amazonS3.putObject(request);
} catch (Throwable t) {
ExceptionUtils.rethrow(t);
}
}
@Override
public void download(DownloadRequest downloadRequest) {
try {
FileUtils.forceMkdirParent(downloadRequest.getTarget());
String fileName = parseFileName(downloadRequest.getFileLocation());
GetObjectRequest getObjectRequest = new GetObjectRequest(this.bucket, fileName);
amazonS3.getObject(getObjectRequest, downloadRequest.getTarget());
} catch (Throwable t) {
ExceptionUtils.rethrow(t);
}
}
/**
* 获取文件元
*
* @param fileLocation 文件位置
*/
@Override
public Optional<FileMeta> fetchFileMeta(FileLocation fileLocation) {
try {
String fileName = parseFileName(fileLocation);
ObjectMetadata objectMetadata = amazonS3.getObjectMetadata(this.bucket, fileName);
return Optional.ofNullable(objectMetadata).map(minioStat -> {
Map<String, Object> metaInfo = Maps.newHashMap();
if (objectMetadata.getRawMetadata() != null) {
metaInfo.putAll(objectMetadata.getRawMetadata());
}
return new FileMeta()
.setLastModifiedTime(objectMetadata.getLastModified())
.setLength(objectMetadata.getContentLength())
.setMetaInfo(metaInfo);
});
} catch (AmazonS3Exception s3Exception) {
String errorCode = s3Exception.getErrorCode();
if (NOT_FOUNT.equalsIgnoreCase(errorCode)) {
return Optional.empty();
}
} catch (Exception oe) {
ExceptionUtils.rethrow(oe);
}
return Optional.empty();
}
private static String parseFileName(FileLocation fileLocation) {
return String.format("%s/%s", fileLocation.getBucket(), fileLocation.getName());
}
/**
* 清理过期文件
*
* @param bucket 桶名
* @param days 日期
*/
@Override
public void cleanExpiredFiles(String bucket, int days) {
/*
使用Minio的管理界面或Minio客户端命令行工具设置对象的生命周期规则。在生命周期规则中定义文件的过期时间。Minio将自动根据设置的规则删除过期的文件。
*/
}
/**
* 释放连接
*/
@Override
public void destroy() {
//minioClient.close();
}
/**
* 初始化minio
*
* @param applicationContext /
*/
@Override
protected void init(ApplicationContext applicationContext) {
Environment environment = applicationContext.getEnvironment();
String endpoint = fetchProperty(environment, TYPE_MINIO, KEY_ENDPOINT);
String bucketName = fetchProperty(environment, TYPE_MINIO, KEY_BUCKET_NAME);
String accessKey = fetchProperty(environment, TYPE_MINIO, ACCESS_KEY);
String secretKey = fetchProperty(environment, TYPE_MINIO, SECRET_KEY);
try {
initOssClient(endpoint, bucketName, accessKey, secretKey);
} catch (Exception e) {
ExceptionUtils.rethrow(e);
}
}
/**
* 创建minio连接并且创建桶
*
* @param endpoint 端口
* @param bucketName 桶名
* @param accessKey 访问密钥
* @param secretKey 秘密密钥
*/
public void initOssClient(String endpoint, String bucketName, String accessKey, String secretKey) {
log.info("[Minio] init OSS by config: endpoint={}, bucketName={}, accessKey={}, secretKey={}", endpoint, bucketName, accessKey, secretKey);
if (StringUtils.isEmpty(bucketName)) {
throw new IllegalArgumentException("'oms.storage.dfs.minio.bucketName' can't be empty, please creat a bucket in minio oss console then config it to powerjob");
}
// 创建凭证对象
BasicAWSCredentials awsCreds = new BasicAWSCredentials(accessKey, secretKey);
// 创建AmazonS3客户端并指定终端节点和凭证
this.amazonS3 = AmazonS3ClientBuilder.standard()
// 当使用 AWS Java SDK 连接到非AWS服务（如MinIO）时，指定区域（Region）是必需的，即使这个区域对于你的MinIO实例并不真正适用。原因在于AWS SDK的客户端构建器需要一个区域来配置其服务端点，即使在连接到本地或第三方S3兼容服务时也是如此。使用 "us-east-1" 作为占位符是很常见的做法，因为它是AWS最常用的区域之一。这不会影响到实际的连接或数据传输，因为真正的服务地址是由你提供的终端节点URL决定的。如果你的代码主要是与MinIO交互，并且不涉及AWS服务，那么这个区域设置只是形式上的要求。
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "us-east-1"))
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.withPathStyleAccessEnabled(true) // 重要：启用路径样式访问
.build();
this.bucket = bucketName;
createBucket(bucketName);
log.info("[Minio] initialize OSS successfully!");
}
/**
* 创建 bucket
*
* @param bucketName 桶名
*/
@SneakyThrows(Exception.class)
public void createBucket(String bucketName) {
// 建议自行创建 bucket，设置好相关的策略
if (bucketExists(bucketName)) {
return;
}
Bucket createBucketResult = amazonS3.createBucket(bucketName);
log.info("[Minio] createBucket successfully, bucketName: {}, createResult: {}", bucketName, createBucketResult);
String policy = "{\n" +
" \"Version\": \"2012-10-17\",\n" +
" \"Statement\": [\n" +
" {\n" +
" \"Action\": [\n" +
" \"s3:GetObject\"\n" +
" ],\n" +
" \"Effect\": \"Allow\",\n" +
" \"Principal\": {\n" +
" \"AWS\": [\n" +
" \"*\"\n" +
" ]\n" +
" },\n" +
" \"Resource\": [\n" +
" \"arn:aws:s3:::" + bucketName + "/*\"\n" +
" ]\n" +
" }\n" +
" ]\n" +
"}";
try {
amazonS3.setBucketPolicy(bucketName, policy);
} catch (Exception e) {
log.warn("[Minio] setBucketPolicy failed, maybe you need to setBucketPolicy by yourself!", e);
}
}
/**
* 判断 bucket是否存在
*
* @param bucketName: 桶名
* @return boolean
*/
@SneakyThrows(Exception.class)
public boolean bucketExists(String bucketName) {
return amazonS3.doesBucketExistV2(bucketName);
}
public static class MinioOssCondition extends PropertyAndOneBeanCondition {
@Override
protected List<String> anyConfigKey() {
return Lists.newArrayList("oms.storage.dfs.minio.endpoint");
}
@Override
protected Class<?> beanType() {
return DFsService.class;
}
}
}

350
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsService.java Normal file
View File

@ -0,0 +1,350 @@
package tech.powerjob.server.persistence.storage.impl;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Conditional;
import org.springframework.core.env.Environment;
import tech.powerjob.common.serialize.JsonUtils;
import tech.powerjob.common.utils.CommonUtils;
import tech.powerjob.common.enums.SwitchableStatus;
import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition;
import tech.powerjob.server.extension.dfs.*;
import tech.powerjob.server.persistence.storage.AbstractDFsService;
import javax.annotation.Priority;
import javax.sql.DataSource;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.sql.*;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* MySQL 特性类似的数据库存储
* PS1. 大文件上传可能会报 max_allowed_packet 不足，可根据参数放开数据库限制 set global max_allowed_packet = 500*1024*1024
* PS2. 官方基于 MySQL 测试，其他数据库使用前请自测，敬请谅解！
* PS3. 数据库并不适合大规模的文件存储，该扩展仅适用于简单业务，大型业务场景请选择其他存储方案（OSS、MongoDB等）
* ********************* 配置项 *********************
* oms.storage.dfs.mysql_series.driver
* oms.storage.dfs.mysql_series.url
* oms.storage.dfs.mysql_series.username
* oms.storage.dfs.mysql_series.password
* oms.storage.dfs.mysql_series.auto_create_table
* oms.storage.dfs.mysql_series.table_name
*
* @author tjq
* @since 2023/8/9
*/
@Slf4j
@Priority(value = Integer.MAX_VALUE - 2)
@Conditional(MySqlSeriesDfsService.MySqlSeriesCondition.class)
public class MySqlSeriesDfsService extends AbstractDFsService {
private DataSource dataSource;
private static final String TYPE_MYSQL = "mysql_series";
/**
* 数据库驱动，MYSQL8 为 com.mysql.cj.jdbc.Driver
*/
private static final String KEY_DRIVER_NAME = "driver";
/**
* 数据库地址，比如 jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai
*/
private static final String KEY_URL = "url";
/**
* 数据库账号，比如 root
*/
private static final String KEY_USERNAME = "username";
/**
* 数据库密码
*/
private static final String KEY_PASSWORD = "password";
/**
* 是否自动建表
*/
private static final String KEY_AUTO_CREATE_TABLE = "auto_create_table";
/**
* 表名
*/
private static final String KEY_TABLE_NAME = "table_name";
/* ********************* SQL region ********************* */
private static final String DEFAULT_TABLE_NAME = "powerjob_files";
private static final String CREATE_TABLE_SQL = "CREATE TABLE\n" +
"IF\n" +
"\tNOT EXISTS %s (\n" +
"\t\t`id` BIGINT NOT NULL AUTO_INCREMENT COMMENT 'ID',\n" +
"\t\t`bucket` VARCHAR ( 255 ) NOT NULL COMMENT '分桶',\n" +
"\t\t`name` VARCHAR ( 255 ) NOT NULL COMMENT '文件名称',\n" +
"\t\t`version` VARCHAR ( 255 ) NOT NULL COMMENT '版本',\n" +
"\t\t`meta` VARCHAR ( 255 ) COMMENT '元数据',\n" +
"\t\t`length` BIGINT NOT NULL COMMENT '长度',\n" +
"\t\t`status` INT NOT NULL COMMENT '状态',\n" +
"\t\t`data` LONGBLOB NOT NULL COMMENT '文件内容',\n" +
"\t\t`extra` VARCHAR ( 255 ) COMMENT '其他信息',\n" +
"\t\t`gmt_create` DATETIME NOT NULL COMMENT '创建时间',\n" +
"\t\t`gmt_modified` DATETIME COMMENT '更新时间',\n" +
"\tPRIMARY KEY ( id ) \n" +
"\t);";
private static final String INSERT_SQL = "insert into %s(bucket, name, version, meta, length, status, data, extra, gmt_create, gmt_modified) values (?,?,?,?,?,?,?,?,?,?);";
private static final String DELETE_SQL = "DELETE FROM %s ";
private static final String QUERY_FULL_SQL = "select * from %s";
private static final String QUERY_META_SQL = "select bucket, name, version, meta, length, status, extra, gmt_create, gmt_modified from %s";
private void deleteByLocation(FileLocation fileLocation) {
String dSQLPrefix = fullSQL(DELETE_SQL);
String dSQL = dSQLPrefix.concat(whereSQL(fileLocation));
executeDelete(dSQL);
}
private void executeDelete(String sql) {
try (Connection con = dataSource.getConnection()) {
con.createStatement().executeUpdate(sql);
} catch (Exception e) {
log.error("[MySqlSeriesDfsService] executeDelete failed, sql: {}", sql);
}
}
@Override
public void store(StoreRequest storeRequest) throws IOException {
Stopwatch sw = Stopwatch.createStarted();
String insertSQL = fullSQL(INSERT_SQL);
FileLocation fileLocation = storeRequest.getFileLocation();
// 覆盖写，写之前先删除
deleteByLocation(fileLocation);
Map<String, Object> meta = Maps.newHashMap();
meta.put("_server_", serverInfo.getIp());
meta.put("_local_file_path_", storeRequest.getLocalFile().getAbsolutePath());
BufferedInputStream bufferedInputStream = new BufferedInputStream(Files.newInputStream(storeRequest.getLocalFile().toPath()));
Date date = new Date(System.currentTimeMillis());
try (Connection con = dataSource.getConnection()) {
PreparedStatement pst = con.prepareStatement(insertSQL);
pst.setString(1, fileLocation.getBucket());
pst.setString(2, fileLocation.getName());
pst.setString(3, "mu");
pst.setString(4, JsonUtils.toJSONString(meta));
pst.setLong(5, storeRequest.getLocalFile().length());
pst.setInt(6, SwitchableStatus.ENABLE.getV());
pst.setBlob(7, bufferedInputStream);
pst.setString(8, null);
pst.setDate(9, date);
pst.setDate(10, date);
pst.execute();
log.info("[MySqlSeriesDfsService] store [{}] successfully, cost: {}", fileLocation, sw);
} catch (Exception e) {
log.error("[MySqlSeriesDfsService] store [{}] failed!", fileLocation);
ExceptionUtils.rethrow(e);
}finally {
bufferedInputStream.close();
}
}
@Override
public void download(DownloadRequest downloadRequest) throws IOException {
Stopwatch sw = Stopwatch.createStarted();
String querySQL = fullSQL(QUERY_FULL_SQL);
FileLocation fileLocation = downloadRequest.getFileLocation();
FileUtils.forceMkdirParent(downloadRequest.getTarget());
try (Connection con = dataSource.getConnection()) {
ResultSet resultSet = con.createStatement().executeQuery(querySQL.concat(whereSQL(fileLocation)));
boolean exist = resultSet.next();
if (!exist) {
log.warn("[MySqlSeriesDfsService] download file[{}] failed due to not exits!", fileLocation);
return;
}
Blob dataBlob = resultSet.getBlob("data");
FileUtils.copyInputStreamToFile(new BufferedInputStream(dataBlob.getBinaryStream()), downloadRequest.getTarget());
log.info("[MySqlSeriesDfsService] download [{}] successfully, cost: {}", fileLocation, sw);
} catch (Exception e) {
log.error("[MySqlSeriesDfsService] download file [{}] failed!", fileLocation, e);
ExceptionUtils.rethrow(e);
}
}
@Override
public Optional<FileMeta> fetchFileMeta(FileLocation fileLocation) throws IOException {
String querySQL = fullSQL(QUERY_META_SQL);
try (Connection con = dataSource.getConnection()) {
ResultSet resultSet = con.createStatement().executeQuery(querySQL.concat(whereSQL(fileLocation)));
boolean exist = resultSet.next();
if (!exist) {
return Optional.empty();
}
FileMeta fileMeta = new FileMeta()
.setLength(resultSet.getLong("length"))
.setLastModifiedTime(resultSet.getDate("gmt_modified"))
.setMetaInfo(JsonUtils.parseMap(resultSet.getString("meta")));
return Optional.of(fileMeta);
} catch (Exception e) {
log.error("[MySqlSeriesDfsService] fetchFileMeta [{}] failed!", fileLocation);
ExceptionUtils.rethrow(e);
}
return Optional.empty();
}
@Override
public void cleanExpiredFiles(String bucket, int days) {
// 虽然官方提供了服务端删除的能力，依然强烈建议用户直接在数据库层面配置清理事件！！！
String dSQLPrefix = fullSQL(DELETE_SQL);
final long targetTs = DateUtils.addDays(new Date(System.currentTimeMillis()), -days).getTime();
final String targetDeleteTime = CommonUtils.formatTime(targetTs);
log.info("[MySqlSeriesDfsService] start to cleanExpiredFiles, targetDeleteTime: {}", targetDeleteTime);
String fSQL = dSQLPrefix.concat(String.format(" where gmt_modified < '%s'", targetDeleteTime));
log.info("[MySqlSeriesDfsService] cleanExpiredFiles SQL: {}", fSQL);
executeDelete(fSQL);
}
@Override
protected void init(ApplicationContext applicationContext) {
Environment env = applicationContext.getEnvironment();
MySQLProperty mySQLProperty = new MySQLProperty()
.setDriver(fetchProperty(env, TYPE_MYSQL, KEY_DRIVER_NAME))
.setUrl(fetchProperty(env, TYPE_MYSQL, KEY_URL))
.setUsername(fetchProperty(env, TYPE_MYSQL, KEY_USERNAME))
.setPassword(fetchProperty(env, TYPE_MYSQL, KEY_PASSWORD))
.setAutoCreateTable(Boolean.TRUE.toString().equalsIgnoreCase(fetchProperty(env, TYPE_MYSQL, KEY_AUTO_CREATE_TABLE)))
;
try {
initDatabase(mySQLProperty);
initTable(mySQLProperty);
} catch (Exception e) {
log.error("[MySqlSeriesDfsService] init datasource failed!", e);
ExceptionUtils.rethrow(e);
}
log.info("[MySqlSeriesDfsService] initialize successfully, THIS_WILL_BE_THE_STORAGE_LAYER.");
}
void initDatabase(MySQLProperty property) {
log.info("[MySqlSeriesDfsService] init datasource by config: {}", property);
HikariConfig config = new HikariConfig();
config.setDriverClassName(property.driver);
config.setJdbcUrl(property.url);
config.setUsername(property.username);
config.setPassword(property.password);
config.setAutoCommit(true);
// 池中最小空闲连接数量
config.setMinimumIdle(2);
// 池中最大连接数量
config.setMaximumPoolSize(32);
dataSource = new HikariDataSource(config);
}
void initTable(MySQLProperty property) throws Exception {
if (property.autoCreateTable) {
String createTableSQL = fullSQL(CREATE_TABLE_SQL);
log.info("[MySqlSeriesDfsService] use create table SQL: {}", createTableSQL);
try (Connection connection = dataSource.getConnection()) {
connection.createStatement().execute(createTableSQL);
log.info("[MySqlSeriesDfsService] auto create table successfully!");
}
}
}
private String fullSQL(String sql) {
return String.format(sql, parseTableName());
}
private String parseTableName() {
// 误删，兼容本地 unit test
if (applicationContext == null) {
return DEFAULT_TABLE_NAME;
}
String tableName = fetchProperty(applicationContext.getEnvironment(), TYPE_MYSQL, KEY_TABLE_NAME);
return StringUtils.isEmpty(tableName) ? DEFAULT_TABLE_NAME : tableName;
}
private static String whereSQL(FileLocation fileLocation) {
return String.format(" where bucket='%s' AND name='%s' ", fileLocation.getBucket(), fileLocation.getName());
}
@Override
public void destroy() throws Exception {
}
@Data
@Accessors(chain = true)
static class MySQLProperty {
private String driver;
private String url;
private String username;
private String password;
private boolean autoCreateTable;
}
public static class MySqlSeriesCondition extends PropertyAndOneBeanCondition {
@Override
protected List<String> anyConfigKey() {
return Lists.newArrayList("oms.storage.dfs.mysql_series.url");
}
@Override
protected Class<?> beanType() {
return DFsService.class;
}
}
}

406
powerjob-server/powerjob-server-persistence/src/main/java/tech/powerjob/server/persistence/storage/impl/PostgresqlSeriesDfsService.java Normal file
View File

@ -0,0 +1,406 @@
package tech.powerjob.server.persistence.storage.impl;
import com.google.common.base.Stopwatch;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import lombok.Data;
import lombok.experimental.Accessors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.context.ApplicationContext;
import org.springframework.context.annotation.Conditional;
import org.springframework.core.env.Environment;
import tech.powerjob.common.enums.SwitchableStatus;
import tech.powerjob.common.serialize.JsonUtils;
import tech.powerjob.common.utils.CommonUtils;
import tech.powerjob.server.common.spring.condition.PropertyAndOneBeanCondition;
import tech.powerjob.server.extension.dfs.*;
import tech.powerjob.server.persistence.storage.AbstractDFsService;
import javax.annotation.Priority;
import javax.sql.DataSource;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.sql.*;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* postgresql 数据库存储，使用的版本是14
* ********************* 配置项 *********************
* oms.storage.dfs.postgresql_series.driver
* oms.storage.dfs.postgresql_series.url
* oms.storage.dfs.postgresql_series.username
* oms.storage.dfs.postgresql_series.password
* oms.storage.dfs.postgresql_series.auto_create_table
* oms.storage.dfs.postgresql_series.table_name
*
* @author jetol
* @since 2024-1-8
*/
@Slf4j
@Priority(value = Integer.MAX_VALUE - 4)
@Conditional(PostgresqlSeriesDfsService.PostgresqlSeriesCondition.class)
public class PostgresqlSeriesDfsService extends AbstractDFsService {
private DataSource dataSource;
private static final String TYPE_POSTGRESQL = "postgresql_series";
/**
* 数据库驱动，Postgresql 为 org.postgresql.Driver
*/
private static final String KEY_DRIVER_NAME = "driver";
/**
* 数据库地址，比如 jdbc:postgresql://localhost:3306/powerjob-daily
*/
private static final String KEY_URL = "url";
/**
* 数据库账号，比如 root
*/
private static final String KEY_USERNAME = "username";
/**
* 数据库密码
*/
private static final String KEY_PASSWORD = "password";
/**
* 是否自动建表
*/
private static final String KEY_AUTO_CREATE_TABLE = "auto_create_table";
/**
* 表名
*/
private static final String KEY_TABLE_NAME = "table_name";
/* ********************* SQL region ********************* */
private static final String DEFAULT_TABLE_NAME = "powerjob_files";
private static final String POWERJOB_FILES_ID_SEQ = "CREATE SEQUENCE if not exists powerjob_files_id_seq\n" +
" START WITH 1\n" +
" INCREMENT BY 1\n" +
" NO MINVALUE\n" +
" NO MAXVALUE\n" +
" CACHE 1;" ;
private static final String CREATE_TABLE_SQL = "CREATE TABLE if not exists powerjob_files (\n" +
" id bigint NOT NULL DEFAULT nextval('powerjob_files_id_seq') PRIMARY KEY,\n" +
" bucket varchar(255) NOT NULL,\n" +
" name varchar(255) NOT NULL,\n" +
" version varchar(255) NOT NULL,\n" +
" meta varchar(255) NULL DEFAULT NULL,\n" +
" length bigint NOT NULL,\n" +
" status int NOT NULL,\n" +
" data bytea NOT NULL,\n" +
" extra varchar(255) NULL DEFAULT NULL,\n" +
" gmt_create timestamp without time zone NOT NULL,\n" +
" gmt_modified timestamp without time zone NULL DEFAULT NULL\n" +
");";
private static final String INSERT_SQL = "insert into %s(bucket, name, version, meta, length, status, data, extra, gmt_create, gmt_modified) values (?,?,?,?,?,?,?,?,?,?);";
private static final String DELETE_SQL = "DELETE FROM %s ";
private static final String QUERY_FULL_SQL = "select * from %s";
private static final String QUERY_META_SQL = "select bucket, name, version, meta, length, status, extra, gmt_create, gmt_modified from %s";
private void deleteByLocation(FileLocation fileLocation) {
String dSQLPrefix = fullSQL(DELETE_SQL);
String dSQL = dSQLPrefix.concat(whereSQL(fileLocation));
executeDelete(dSQL);
}
private void executeDelete(String sql) {
try (Connection con = dataSource.getConnection()) {
con.createStatement().executeUpdate(sql);
} catch (Exception e) {
log.error("[PostgresqlSeriesDfsService] executeDelete failed, sql: {}", sql);
}
}
@Override
public void store(StoreRequest storeRequest) throws IOException, SQLException {
Stopwatch sw = Stopwatch.createStarted();
String insertSQL = fullSQL(INSERT_SQL);
FileLocation fileLocation = storeRequest.getFileLocation();
// 覆盖写，写之前先删除
deleteByLocation(fileLocation);
Map<String, Object> meta = Maps.newHashMap();
meta.put("_server_", serverInfo.getIp());
meta.put("_local_file_path_", storeRequest.getLocalFile().getAbsolutePath());
BufferedInputStream bufferedInputStream = new BufferedInputStream(Files.newInputStream(storeRequest.getLocalFile().toPath()));
Date date = new Date(System.currentTimeMillis());
Connection con =null;
PreparedStatement pst =null;
try {
con = dataSource.getConnection();
//pg库提示报错：org.postgresql.util.PSQLException: Large Objects may not be used in auto-commit mode.
con.setAutoCommit(false);
pst = con.prepareStatement(insertSQL);
pst.setString(1, fileLocation.getBucket());
pst.setString(2, fileLocation.getName());
pst.setString(3, "mu");
pst.setString(4, JsonUtils.toJSONString(meta));
pst.setLong(5, storeRequest.getLocalFile().length());
pst.setInt(6, SwitchableStatus.ENABLE.getV());
//PreparedStatement类并没有提供setBlob方法来直接设置BYTEA类型字段，因为PostgreSQL不支持JDBC中的java.sql.Blob接口
// pst.setBlob(7, bufferedInputStream);org.postgresql.util.PSQLException: ERROR: column "data" is of type bytea but expression is of type bigint
pst.setBytes(7, bufferedInputStreamToByteArray(bufferedInputStream));
pst.setString(8, null);
pst.setDate(9, date);
pst.setDate(10, date);
pst.execute();
con.commit();
log.info("[PostgresqlSeriesDfsService] store [{}] successfully, cost: {}", fileLocation, sw);
} catch (Exception e) {
if(con != null){
con.rollback();
}
log.error("[PostgresqlSeriesDfsService] store [{}] failed!", fileLocation, e);
ExceptionUtils.rethrow(e);
}finally {
if(con != null){
//设置回来，恢复自动提交模式
con.setAutoCommit(true);
con.close();
}
if(null != pst){
pst.close();
}
bufferedInputStream.close();
}
}
/**
* 上面已经有异常处理，这里直接往上抛
* @param bis
* @return
* @throws IOException
*/
public static byte[] bufferedInputStreamToByteArray(BufferedInputStream bis) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
if(null == bis ){
return null;
}
// 创建缓冲区
byte[] buffer = new byte[1024];
int read;
// 读取流中的数据并写入到ByteArrayOutputStream
while ((read = bis.read(buffer)) != -1) {
baos.write(buffer, 0, read);
}
// 关闭输入流
bis.close();
// 转换为字节数组并返回
return baos.toByteArray();
}
@Override
public void download(DownloadRequest downloadRequest) throws IOException {
Stopwatch sw = Stopwatch.createStarted();
String querySQL = fullSQL(QUERY_FULL_SQL);
FileLocation fileLocation = downloadRequest.getFileLocation();
FileUtils.forceMkdirParent(downloadRequest.getTarget());
try (Connection con = dataSource.getConnection()) {
ResultSet resultSet = con.createStatement().executeQuery(querySQL.concat(whereSQL(fileLocation)));
boolean exist = resultSet.next();
if (!exist) {
log.warn("[PostgresqlSeriesDfsService] download file[{}] failed due to not exits!", fileLocation);
return;
}
// 在 PostgreSQL 中，bytea 类型的数据并不直接映射为 JDBC 的 Blob 类型。相反，bytea 数据应当被处理为字节数组 (byte[]) 而不是 Blob 对象
try {
byte[] dataBytes = resultSet.getBytes("data");
try (FileOutputStream fos = new FileOutputStream(downloadRequest.getTarget())) {
fos.write(dataBytes);
}
} catch (Exception ignore) {
// 测试发现会报错 报错“不良的类型值 long”；但并未有用户反馈问题，暂时保留老写法，可能是不同DB获取方式不同？
Blob dataBlob = resultSet.getBlob("data");
FileUtils.copyInputStreamToFile(new BufferedInputStream(dataBlob.getBinaryStream()), downloadRequest.getTarget());
}
log.info("[PostgresqlSeriesDfsService] download [{}] successfully, cost: {}", fileLocation, sw);
} catch (Exception e) {
log.error("[PostgresqlSeriesDfsService] download file [{}] failed!", fileLocation, e);
ExceptionUtils.rethrow(e);
}
}
@Override
public Optional<FileMeta> fetchFileMeta(FileLocation fileLocation) {
String querySQL = fullSQL(QUERY_META_SQL);
try (Connection con = dataSource.getConnection()) {
ResultSet resultSet = con.createStatement().executeQuery(querySQL.concat(whereSQL(fileLocation)));
boolean exist = resultSet.next();
if (!exist) {
return Optional.empty();
}
FileMeta fileMeta = new FileMeta()
.setLength(resultSet.getLong("length"))
.setLastModifiedTime(resultSet.getDate("gmt_modified"))
.setMetaInfo(JsonUtils.parseMap(resultSet.getString("meta")));
return Optional.of(fileMeta);
} catch (Exception e) {
log.error("[PostgresqlSeriesDfsService] fetchFileMeta [{}] failed!", fileLocation);
ExceptionUtils.rethrow(e);
}
return Optional.empty();
}
@Override
public void cleanExpiredFiles(String bucket, int days) {
// 虽然官方提供了服务端删除的能力，依然强烈建议用户直接在数据库层面配置清理事件！！！
String dSQLPrefix = fullSQL(DELETE_SQL);
final long targetTs = DateUtils.addDays(new Date(System.currentTimeMillis()), -days).getTime();
final String targetDeleteTime = CommonUtils.formatTime(targetTs);
log.info("[PostgresqlSeriesDfsService] start to cleanExpiredFiles, targetDeleteTime: {}", targetDeleteTime);
String fSQL = dSQLPrefix.concat(String.format(" where gmt_modified < '%s'", targetDeleteTime));
log.info("[PostgresqlSeriesDfsService] cleanExpiredFiles SQL: {}", fSQL);
executeDelete(fSQL);
}
@Override
protected void init(ApplicationContext applicationContext) {
Environment env = applicationContext.getEnvironment();
PostgresqlProperty postgresqlProperty = new PostgresqlProperty()
.setDriver(fetchProperty(env, TYPE_POSTGRESQL, KEY_DRIVER_NAME))
.setUrl(fetchProperty(env, TYPE_POSTGRESQL, KEY_URL))
.setUsername(fetchProperty(env, TYPE_POSTGRESQL, KEY_USERNAME))
.setPassword(fetchProperty(env, TYPE_POSTGRESQL, KEY_PASSWORD))
.setAutoCreateTable(Boolean.TRUE.toString().equalsIgnoreCase(fetchProperty(env, TYPE_POSTGRESQL, KEY_AUTO_CREATE_TABLE)))
;
try {
initDatabase(postgresqlProperty);
initTable(postgresqlProperty);
} catch (Exception e) {
log.error("[PostgresqlSeriesDfsService] init datasource failed!", e);
ExceptionUtils.rethrow(e);
}
log.info("[PostgresqlSeriesDfsService] initialize successfully, THIS_WILL_BE_THE_STORAGE_LAYER.");
}
void initDatabase(PostgresqlProperty property) {
log.info("[PostgresqlSeriesDfsService] init datasource by config: {}", property);
HikariConfig config = new HikariConfig();
config.setDriverClassName(StringUtils.isEmpty(property.driver) ? "org.postgresql.Driver" : property.driver);
config.setJdbcUrl(property.url);
config.setUsername(property.username);
config.setPassword(property.password);
config.setAutoCommit(true);
// 池中最小空闲连接数量
config.setMinimumIdle(2);
// 池中最大连接数量
config.setMaximumPoolSize(32);
dataSource = new HikariDataSource(config);
}
void initTable(PostgresqlProperty property) throws Exception {
if (property.autoCreateTable) {
String powerjobFilesIdSeq = fullSQL(POWERJOB_FILES_ID_SEQ);
String createTableSQL = fullSQL(CREATE_TABLE_SQL);
log.info("[PostgresqlSeriesDfsService] use create table SQL: {}", createTableSQL);
try (Connection connection = dataSource.getConnection()) {
connection.createStatement().execute(powerjobFilesIdSeq);
connection.createStatement().execute(createTableSQL);
log.info("[PostgresqlSeriesDfsService] auto create table successfully!");
}
}
}
private String fullSQL(String sql) {
return String.format(sql, parseTableName());
}
private String parseTableName() {
// 误删，兼容本地 unit test
if (applicationContext == null) {
return DEFAULT_TABLE_NAME;
}
String tableName = fetchProperty(applicationContext.getEnvironment(), TYPE_POSTGRESQL, KEY_TABLE_NAME);
return StringUtils.isEmpty(tableName) ? DEFAULT_TABLE_NAME : tableName;
}
private static String whereSQL(FileLocation fileLocation) {
return String.format(" where bucket='%s' AND name='%s' ", fileLocation.getBucket(), fileLocation.getName());
}
@Override
public void destroy() throws Exception {
}
@Data
@Accessors(chain = true)
static class PostgresqlProperty {
private String driver;
private String url;
private String username;
private String password;
private boolean autoCreateTable;
}
public static class PostgresqlSeriesCondition extends PropertyAndOneBeanCondition {
@Override
protected List<String> anyConfigKey() {
return Lists.newArrayList("oms.storage.dfs.postgresql_series.url");
}
@Override
protected Class<?> beanType() {
return DFsService.class;
}
}
}

88
powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AbstractDfsServiceTest.java Normal file
View File

@ -0,0 +1,88 @@
package tech.powerjob.server.persistence.storage.impl;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import tech.powerjob.common.serialize.JsonUtils;
import tech.powerjob.server.common.utils.OmsFileUtils;
import tech.powerjob.server.extension.dfs.*;
import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.Optional;
import java.util.concurrent.ThreadLocalRandom;
/**
* AbstractDfsServiceTest
*
* @author tjq
* @since 2023/7/30
*/
@Slf4j
public abstract class AbstractDfsServiceTest {
private static final String BUCKET = "pj_test";
abstract protected Optional<DFsService> fetchService();
@Test
void testBaseFileOperation() throws Exception {
Optional<DFsService> aliOssServiceOpt = fetchService();
if (!aliOssServiceOpt.isPresent()) {
return;
}
DFsService aliOssService = aliOssServiceOpt.get();
String content = "wlcgyqsl".concat(String.valueOf(ThreadLocalRandom.current().nextLong()));
String temporarySourcePath = OmsFileUtils.genTemporaryWorkPath() + "source.txt";
String temporaryDownloadPath = OmsFileUtils.genTemporaryWorkPath() + "download.txt";
log.info("[testBaseFileOperation] temporarySourcePath: {}", temporarySourcePath);
File sourceFile = new File(temporarySourcePath);
FileUtils.forceMkdirParent(sourceFile);
OmsFileUtils.string2File(content, sourceFile);
FileLocation fileLocation = new FileLocation().setBucket(BUCKET).setName(String.format("test_%d.txt", ThreadLocalRandom.current().nextLong()));
StoreRequest storeRequest = new StoreRequest()
.setFileLocation(fileLocation)
.setLocalFile(sourceFile);
// 存储
aliOssService.store(storeRequest);
// 读取 meta
Optional<FileMeta> metaOpt = aliOssService.fetchFileMeta(fileLocation);
assert metaOpt.isPresent();
log.info("[testBaseFileOperation] file meta: {}", JsonUtils.toJSONString(metaOpt.get()));
// 下载
log.info("[testBaseFileOperation] temporaryDownloadPath: {}", temporaryDownloadPath);
File downloadFile = new File(temporaryDownloadPath);
DownloadRequest downloadRequest = new DownloadRequest()
.setFileLocation(fileLocation)
.setTarget(downloadFile);
aliOssService.download(downloadRequest);
String downloadFileContent = FileUtils.readFileToString(downloadFile, StandardCharsets.UTF_8);
log.info("[testBaseFileOperation] download content: {}", downloadFileContent);
assert downloadFileContent.equals(content);
// 定时清理，只是执行，不校验
aliOssService.cleanExpiredFiles(BUCKET, 3);
}
@Test
void testFileNotExist() throws Exception {
Optional<DFsService> aliOssServiceOpt = fetchService();
if (!aliOssServiceOpt.isPresent()) {
return;
}
Optional<FileMeta> metaOpt = aliOssServiceOpt.get().fetchFileMeta(new FileLocation().setBucket("tjq").setName("yhz"));
assert !metaOpt.isPresent();
}
}

49
powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/AliOssServiceTest.java Normal file
View File

@ -0,0 +1,49 @@
package tech.powerjob.server.persistence.storage.impl;
import com.aliyun.oss.common.utils.AuthUtils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import tech.powerjob.server.extension.dfs.DFsService;
import java.util.Optional;
/**
* test AliOSS
*
* @author tjq
* @since 2023/7/30
*/
@Slf4j
class AliOssServiceTest extends AbstractDfsServiceTest {
private static final String BUCKET = "power-job";
/**
* 依赖阿里云账号密码测试，为了保证单测在其他环境也能通过，如果发现不存在配置则直接跳过
* @return AliOssService
*/
@Override
protected Optional<DFsService> fetchService() {
String accessKeyId = StringUtils.trim(System.getenv(AuthUtils.ACCESS_KEY_ENV_VAR));
String secretAccessKey = StringUtils.trim(System.getenv(AuthUtils.SECRET_KEY_ENV_VAR));
String bucket = Optional.ofNullable(System.getenv("POWERJOB_OSS_BUEKCT")).orElse(BUCKET);
log.info("[AliOssServiceTest] ak: {}, sk: {}", accessKeyId, secretAccessKey);
if (StringUtils.isAnyEmpty(accessKeyId, secretAccessKey)) {
return Optional.empty();
}
try {
AliOssService aliOssService = new AliOssService();
aliOssService.initOssClient("oss-cn-beijing.aliyuncs.com", bucket, AliOssService.CredentialType.ENV.getCode(), null, null, null);
return Optional.of(aliOssService);
} catch (Exception e) {
ExceptionUtils.rethrow(e);
}
return Optional.empty();
}
}

32
powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/GridFsServiceTest.java Normal file
View File

@ -0,0 +1,32 @@
package tech.powerjob.server.persistence.storage.impl;
import lombok.extern.slf4j.Slf4j;
import tech.powerjob.server.common.utils.TestUtils;
import tech.powerjob.server.extension.dfs.DFsService;
import java.util.Optional;
/**
* test GridFS
*
* @author tjq
* @since 2023/7/30
*/
@Slf4j
class GridFsServiceTest extends AbstractDfsServiceTest {
@Override
protected Optional<DFsService> fetchService() {
Object mongoUri = TestUtils.fetchTestConfig().get(TestUtils.KEY_MONGO_URI);
if (mongoUri == null) {
log.info("[GridFsServiceTest] mongoUri is null, skip load!");
return Optional.empty();
}
GridFsService gridFsService = new GridFsService();
gridFsService.initMongo(String.valueOf(mongoUri));
return Optional.of(gridFsService);
}
}

30
powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/MinioOssServiceTest.java Normal file
View File

@ -0,0 +1,30 @@
package tech.powerjob.server.persistence.storage.impl;
import lombok.extern.slf4j.Slf4j;
import tech.powerjob.server.extension.dfs.DFsService;
import java.util.Optional;
/**
* MinioOssServiceTest
* 测试需要先本地部署 minio，因此捕获异常，失败也不阻断测试
*
* @author tjq
* @since 2024/2/26
*/
@Slf4j
class MinioOssServiceTest extends AbstractDfsServiceTest {
@Override
protected Optional<DFsService> fetchService() {
try {
MinioOssService aliOssService = new MinioOssService();
aliOssService.initOssClient("http://192.168.124.23:9000", "pj2","testAk", "testSktestSktestSk");
return Optional.of(aliOssService);
} catch (Exception e) {
// 仅异常提醒
log.error("[MinioOssServiceTest] test exception!", e);
}
return Optional.empty();
}
}

42
powerjob-server/powerjob-server-persistence/src/test/java/tech/powerjob/server/persistence/storage/impl/MySqlSeriesDfsServiceTest.java Normal file
View File

@ -0,0 +1,42 @@
package tech.powerjob.server.persistence.storage.impl;
import tech.powerjob.common.utils.NetUtils;
import tech.powerjob.server.extension.dfs.DFsService;
import java.util.Optional;
/**
* MySqlSeriesDfsServiceTest
*
* @author tjq
* @since 2023/8/10
*/
class MySqlSeriesDfsServiceTest extends AbstractDfsServiceTest {
@Override
protected Optional<DFsService> fetchService() {
boolean dbAvailable = NetUtils.checkIpPortAvailable("127.0.0.1", 3306);
if (dbAvailable) {
MySqlSeriesDfsService mySqlSeriesDfsService = new MySqlSeriesDfsService();
try {
MySqlSeriesDfsService.MySQLProperty mySQLProperty = new MySqlSeriesDfsService.MySQLProperty()
.setDriver("com.mysql.cj.jdbc.Driver")
.setUrl("jdbc:mysql://localhost:3306/powerjob-daily?useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai")
.setUsername("root")
.setAutoCreateTable(true)
.setPassword("No1Bug2Please3!");
mySqlSeriesDfsService.initDatabase(mySQLProperty);
mySqlSeriesDfsService.initTable(mySQLProperty);
return Optional.of(mySqlSeriesDfsService);
} catch (Exception e) {
e.printStackTrace();
}
}
return Optional.empty();
}
}