feat:updates
This commit is contained in:
parent
3621689ecb
commit
3f561bd76f
|
|
@ -10,8 +10,13 @@ import java.util.ArrayList;
|
||||||
import java.util.LinkedHashSet;
|
import java.util.LinkedHashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.UUID;
|
||||||
|
import java.util.concurrent.CompletableFuture;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.dromara.art.domain.bo.*;
|
import org.dromara.art.domain.bo.*;
|
||||||
import org.dromara.art.domain.vo.ArtSchoolImportDetailVo;
|
import org.dromara.art.domain.vo.ArtSchoolImportDetailVo;
|
||||||
import org.dromara.art.domain.vo.ArtSchoolJsonImportPreviewVo;
|
import org.dromara.art.domain.vo.ArtSchoolJsonImportPreviewVo;
|
||||||
|
|
@ -33,6 +38,7 @@ import org.springframework.web.bind.annotation.RestController;
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/art/test")
|
@RequestMapping("/art/test")
|
||||||
|
@Slf4j
|
||||||
public class ArtTestController extends BaseController {
|
public class ArtTestController extends BaseController {
|
||||||
|
|
||||||
private final IArtSchoolService artSchoolService;
|
private final IArtSchoolService artSchoolService;
|
||||||
|
|
@ -43,10 +49,13 @@ public class ArtTestController extends BaseController {
|
||||||
|
|
||||||
@GetMapping("/importSchoolByJson")
|
@GetMapping("/importSchoolByJson")
|
||||||
public R<ArtSchoolJsonImportPreviewVo> importSchoolByJson(
|
public R<ArtSchoolJsonImportPreviewVo> importSchoolByJson(
|
||||||
@RequestParam(defaultValue = "20") Integer previewSize
|
@RequestParam(defaultValue = "20") Integer previewSize,
|
||||||
|
@RequestParam(defaultValue = "true") Boolean async,
|
||||||
|
@RequestParam(defaultValue = "8") Integer threadCount
|
||||||
) {
|
) {
|
||||||
int safePreviewSize =
|
int safePreviewSize =
|
||||||
previewSize == null ? 20 : Math.min(Math.max(previewSize, 1), 200);
|
previewSize == null ? 20 : Math.min(Math.max(previewSize, 1), 200);
|
||||||
|
int safeThreadCount = resolveThreadCount(threadCount);
|
||||||
Path sourceDir;
|
Path sourceDir;
|
||||||
if (StringUtils.isNotBlank(schoolJsonDir)) {
|
if (StringUtils.isNotBlank(schoolJsonDir)) {
|
||||||
sourceDir = Path.of(schoolJsonDir.trim());
|
sourceDir = Path.of(schoolJsonDir.trim());
|
||||||
|
|
@ -69,6 +78,7 @@ public class ArtTestController extends BaseController {
|
||||||
|
|
||||||
List<ArtSchoolImportDetailVo> details = new ArrayList<>();
|
List<ArtSchoolImportDetailVo> details = new ArrayList<>();
|
||||||
List<ArtSchoolSubmitBo> submitPreviewList = new ArrayList<>();
|
List<ArtSchoolSubmitBo> submitPreviewList = new ArrayList<>();
|
||||||
|
List<ArtSchoolSubmitBo> submitAllList = new ArrayList<>();
|
||||||
Set<Integer> fileUnivIdSet = new LinkedHashSet<>();
|
Set<Integer> fileUnivIdSet = new LinkedHashSet<>();
|
||||||
Set<String> fileMainCodeSet = new LinkedHashSet<>();
|
Set<String> fileMainCodeSet = new LinkedHashSet<>();
|
||||||
|
|
||||||
|
|
@ -172,26 +182,10 @@ public class ArtTestController extends BaseController {
|
||||||
submitPreviewList.add(toPreviewSubmitBo(submitBo));
|
submitPreviewList.add(toPreviewSubmitBo(submitBo));
|
||||||
}
|
}
|
||||||
|
|
||||||
// 预留真实入库调用(当前按要求先不入库)
|
submitAllList.add(submitBo);
|
||||||
// artSchoolService.insertWithDetailByBo(submitBo);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (ArtSchoolSubmitBo artSchoolSubmitBo : submitPreviewList) {
|
|
||||||
ArtSchoolBo artSchoolBo = new ArtSchoolBo();
|
|
||||||
artSchoolBo.setMainName(artSchoolSubmitBo.getSchool().getMainName());
|
|
||||||
List<ArtSchoolVo> artSchoolVos = artSchoolService.queryList(artSchoolBo);
|
|
||||||
System.out.println("处理院校信息:" + artSchoolSubmitBo.getSchool().getMainName());
|
|
||||||
if (!artSchoolVos.isEmpty()) {
|
|
||||||
// 修改
|
|
||||||
artSchoolSubmitBo
|
|
||||||
.getSchool()
|
|
||||||
.setSchoolId(artSchoolVos.get(0).getSchoolId());
|
|
||||||
artSchoolService.updateWithDetailByBo(artSchoolSubmitBo);
|
|
||||||
} else {
|
|
||||||
artSchoolService.insertWithDetailByBo(artSchoolSubmitBo);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
previewVo.setReadyCount(readyCount);
|
previewVo.setReadyCount(readyCount);
|
||||||
previewVo.setDuplicateInDbCount(duplicateInDbCount);
|
previewVo.setDuplicateInDbCount(duplicateInDbCount);
|
||||||
previewVo.setDuplicateInFileCount(duplicateInFileCount);
|
previewVo.setDuplicateInFileCount(duplicateInFileCount);
|
||||||
|
|
@ -199,12 +193,127 @@ public class ArtTestController extends BaseController {
|
||||||
previewVo.setReadFailCount(readFailCount);
|
previewVo.setReadFailCount(readFailCount);
|
||||||
previewVo.setDetails(details);
|
previewVo.setDetails(details);
|
||||||
previewVo.setSubmitPreviewList(submitPreviewList);
|
previewVo.setSubmitPreviewList(submitPreviewList);
|
||||||
previewVo.setServiceCallHint(
|
previewVo.setSubmitTotalCount(submitAllList.size());
|
||||||
"已预留调用: artSchoolService.insertWithDetailByBo(submitBo)"
|
previewVo.setThreadCount(safeThreadCount);
|
||||||
);
|
previewVo.setAsync(Boolean.TRUE.equals(async));
|
||||||
|
if (CollUtil.isEmpty(submitAllList)) {
|
||||||
|
previewVo.setServiceCallHint("未发现可入库数据");
|
||||||
|
return R.ok(previewVo);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Boolean.TRUE.equals(async)) {
|
||||||
|
String taskId = UUID.randomUUID().toString();
|
||||||
|
previewVo.setAsyncTaskId(taskId);
|
||||||
|
previewVo.setServiceCallHint(
|
||||||
|
"异步任务已提交: taskId=" +
|
||||||
|
taskId +
|
||||||
|
",预计入库数量=" +
|
||||||
|
submitAllList.size()
|
||||||
|
);
|
||||||
|
CompletableFuture.runAsync(() -> {
|
||||||
|
ImportStats stats = importWithThreadPool(taskId, submitAllList, safeThreadCount);
|
||||||
|
log.info(
|
||||||
|
"学校JSON导入任务完成 taskId={}, total={}, success={}, fail={}",
|
||||||
|
taskId,
|
||||||
|
stats.total,
|
||||||
|
stats.success,
|
||||||
|
stats.fail
|
||||||
|
);
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
ImportStats stats = importWithThreadPool("sync", submitAllList, safeThreadCount);
|
||||||
|
previewVo.setImportSuccessCount(stats.success);
|
||||||
|
previewVo.setImportFailCount(stats.fail);
|
||||||
|
previewVo.setServiceCallHint(
|
||||||
|
"入库完成: total=" +
|
||||||
|
stats.total +
|
||||||
|
", success=" +
|
||||||
|
stats.success +
|
||||||
|
", fail=" +
|
||||||
|
stats.fail
|
||||||
|
);
|
||||||
|
}
|
||||||
return R.ok(previewVo);
|
return R.ok(previewVo);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private int resolveThreadCount(Integer threadCount) {
|
||||||
|
int cpu = Runtime.getRuntime().availableProcessors();
|
||||||
|
int defaultThreads = Math.min(8, Math.max(2, cpu));
|
||||||
|
int maxThreads = Math.max(2, cpu * 2);
|
||||||
|
if (threadCount == null) {
|
||||||
|
return defaultThreads;
|
||||||
|
}
|
||||||
|
return Math.min(Math.max(threadCount, 1), maxThreads);
|
||||||
|
}
|
||||||
|
|
||||||
|
private ImportStats importWithThreadPool(
|
||||||
|
String taskId,
|
||||||
|
List<ArtSchoolSubmitBo> submitAllList,
|
||||||
|
int threadCount
|
||||||
|
) {
|
||||||
|
ExecutorService executor = Executors.newFixedThreadPool(threadCount);
|
||||||
|
List<CompletableFuture<Boolean>> futures = new ArrayList<>(submitAllList.size());
|
||||||
|
for (ArtSchoolSubmitBo submitBo : submitAllList) {
|
||||||
|
futures.add(
|
||||||
|
CompletableFuture.supplyAsync(() -> importSingle(taskId, submitBo), executor)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
int success = 0;
|
||||||
|
int fail = 0;
|
||||||
|
for (CompletableFuture<Boolean> future : futures) {
|
||||||
|
try {
|
||||||
|
if (Boolean.TRUE.equals(future.join())) {
|
||||||
|
success++;
|
||||||
|
} else {
|
||||||
|
fail++;
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
fail++;
|
||||||
|
log.warn("学校JSON导入任务失败 taskId={}, error={}", taskId, e.getMessage(), e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
executor.shutdown();
|
||||||
|
return new ImportStats(submitAllList.size(), success, fail);
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean importSingle(String taskId, ArtSchoolSubmitBo artSchoolSubmitBo) {
|
||||||
|
try {
|
||||||
|
ArtSchoolBo artSchoolBo = new ArtSchoolBo();
|
||||||
|
artSchoolBo.setMainName(artSchoolSubmitBo.getSchool().getMainName());
|
||||||
|
List<ArtSchoolVo> artSchoolVos = artSchoolService.queryList(artSchoolBo);
|
||||||
|
if (!artSchoolVos.isEmpty()) {
|
||||||
|
artSchoolSubmitBo
|
||||||
|
.getSchool()
|
||||||
|
.setSchoolId(artSchoolVos.get(0).getSchoolId());
|
||||||
|
artSchoolService.updateWithDetailByBo(artSchoolSubmitBo);
|
||||||
|
} else {
|
||||||
|
artSchoolService.insertWithDetailByBo(artSchoolSubmitBo);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn(
|
||||||
|
"学校JSON导入任务单条失败 taskId={}, name={}, error={}",
|
||||||
|
taskId,
|
||||||
|
artSchoolSubmitBo.getSchool().getMainName(),
|
||||||
|
e.getMessage(),
|
||||||
|
e
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static class ImportStats {
|
||||||
|
private final int total;
|
||||||
|
private final int success;
|
||||||
|
private final int fail;
|
||||||
|
|
||||||
|
private ImportStats(int total, int success, int fail) {
|
||||||
|
this.total = total;
|
||||||
|
this.success = success;
|
||||||
|
this.fail = fail;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private List<Path> listJsonFiles(Path sourceDir) {
|
private List<Path> listJsonFiles(Path sourceDir) {
|
||||||
try (Stream<Path> stream = Files.list(sourceDir)) {
|
try (Stream<Path> stream = Files.list(sourceDir)) {
|
||||||
return stream
|
return stream
|
||||||
|
|
|
||||||
|
|
@ -61,9 +61,38 @@ public class ArtSchoolJsonImportPreviewVo implements Serializable {
|
||||||
*/
|
*/
|
||||||
private List<ArtSchoolSubmitBo> submitPreviewList;
|
private List<ArtSchoolSubmitBo> submitPreviewList;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 本次可入库总数
|
||||||
|
*/
|
||||||
|
private Integer submitTotalCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 本次导入线程数
|
||||||
|
*/
|
||||||
|
private Integer threadCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 是否异步执行
|
||||||
|
*/
|
||||||
|
private Boolean async;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 异步任务 ID(仅异步时返回)
|
||||||
|
*/
|
||||||
|
private String asyncTaskId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导入成功数量(同步执行时返回)
|
||||||
|
*/
|
||||||
|
private Integer importSuccessCount;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 导入失败数量(同步执行时返回)
|
||||||
|
*/
|
||||||
|
private Integer importFailCount;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 预留调用说明
|
* 预留调用说明
|
||||||
*/
|
*/
|
||||||
private String serviceCallHint;
|
private String serviceCallHint;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue