oracle 圣何塞 debian arm 4C 24G
JDK 21, 不能在本地上复现 代理程序在 服务器上
htop 观察到 是某一个核心突然满载 然后就卡住了
public class Main {
static Log log = Log.get();
static final Gson gson = new Gson();
static List<Image> images = null;
static int maxThreads = 100; // 控制固定线程池的大小
static ExecutorService virtualThreadPool = Executors.newFixedThreadPool(maxThreads, Thread.ofVirtual().factory());
static String filePath = "/root/java_work/imglist/";
// 失败 list
static List<String> failList = Lists.newArrayList();
public static void main(String[] args) {
OkHttpUtils.setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("127.0.0.1", 6789)));
List<String> types = List.of(".jpg", ".png");
try {
images = gson.fromJson(new FileReader("output-2024-4-16.json"), new TypeToken<List<Image>>(){}.getType());
} catch (IOException e) {
log.info("读取文件失败: {}", e.getMessage());
return;
}
log.info("加载到的图片数量: {}", images.size());
images.forEach(image -> virtualThreadPool.submit(() -> {
for (String type : types) {
String url = convertPreviewToImageUrl(image.getHref(), type);
if (attemptToDownloadImage(url,0)) {
image.setSourceUrl(url); // 更新 Image 对象
break;
}
}
}));
virtualThreadPool.shutdown();
while (!virtualThreadPool.isTerminated()) {
Thread.onSpinWait();
}
// 将更新后的 images 列表写回到 JSON 文件
writeImagesToJson(images, "output-2024-4-16-ok.json");
writeImagesToJson(failList, "output-2024-4-16-fail.json");
}
private static boolean attemptToDownloadImage(String url, int retryCount) {
if (retryCount >= 3) {
log.info("重试次数过多,放弃下载: {}", url);
failList.add(url);
return false;
}
try (Response response = OkHttpUtils.get(url, Headers.of("Connection", "close"))) {
switch (response.code()) {
case 200:
log.info("成功下载图片: {}", url);
byte[] bytes = Objects.requireNonNull(response.body()).bytes();
writeImageToFile(bytes, url.substring(url.lastIndexOf('/') + 1));
return true;
case 404:
log.info("图片不存在: {}", url);
return false;
case 429:
log.info("请求过于频繁,需要稍后重试: {}", url);
handleRateLimiting();
return attemptToDownloadImage(url, retryCount + 1);
default:
log.info("其他 HTTP 响应: {}", response.code());
return false;
}
} catch (Exception e) {
log.error("请求图片时出错: {}", e.fillInStackTrace());
handleRateLimiting();
return attemptToDownloadImage(url, retryCount + 1);
}
}
private static void handleRateLimiting() {
try {
log.info("等待 5 秒后重试");
Thread.sleep(5000); // 延迟 5 秒后重试
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
public static String convertPreviewToImageUrl(String previewUrl, String type) {
String id = previewUrl.substring(previewUrl.lastIndexOf('/') + 1);
return String.format("https://w.wallhaven.cc/full/%s/wallhaven-%s%s", id.substring(0, 2), id, type);
}
public static void writeImagesToJson(List<?> objects,String fileName) {
try (FileWriter writer = new FileWriter(fileName)) {
gson.toJson(objects, writer);
} catch (IOException e) {
log.error("写入文件时出错: {}", e.getMessage());
}
}
public static void writeImageToFile(byte[] bytes, String fileName) {
try (FileOutputStream fos = new FileOutputStream(filePath+fileName)) {
fos.write(bytes);
log.info("成功写入文件: {}", fileName);
} catch (IOException e) {
log.error("写入文件时出错: {}", e.getMessage());
}
}
}
https://gist.github.com/dnslin/fe657f9df08f4286c197c5e9e5fd6a51
1
blankmiss OP 我尝试削减过 线程池的数量 减到了 6 个 都会卡住
|
2
blankmiss OP 但是我改成线程池就不会出现这种情况
|
3
sagaxu 219 天前 1
1. 很多第三方库甚至 JDK 库尚不支持虚拟线程
2. 用 JFR 记录观察是哪个地方卡住了 3. onSpinWait 换成 Thread.sleep 试试,busy-waiting 不宜等待太久 |
5
zizon 218 天前
failList 换线程安全的看看?
|