工程(ci远端化·修复): mac_setup Godot URL模板修正(双stable+漏v前缀致404)+镜像假zip校验(200错误页体当失败, unzip -tq守卫双处)+chunk_dl.py收编(分片并行断点续传下载器,已实证方案仓库化)

This commit is contained in:
DKChan committed 2026-11-03 04:16:48 +08:00
1 parent f5c3fa935d
commit fb5f2d42e5
2 files changed
+96 -4

No files matched your search

+88
View File
@@ -0,0 +1,88 @@
#!/usr/bin/env python3
r"""chunk_dl.py — 分片并行下载器(断点续传),2026-11-03 CI 迁移批收编。
背景:GitHub 直连在本项目两台机器均不可用,加速镜像站速度慢且会掐连接/回错误页。
本件把大文件切成 N 片经镜像并行拉取(单片断了自动续传重试),全部齐后拼合。
已实证:74MB Godot Linux 包约 15 分钟拉完(2026-11-03 mac_setup 依赖此件拉 Godot macOS 包)。
用法: python3 tools/ci/chunk_dl.py <URL> <输出路径> [分片数=16]
镜像回落顺序:直连 → gh-proxy.net → gh-proxy.com(HEAD 探测 Content-Length 决定实际用的源)。
校验:分片按字节范围落盘、按序拼合;zip 结构校验由调用方(mac_setup.sh 的 unzip -tq)承担。
退出码:0=成功 1=失败。
"""
import concurrent.futures as cf
import os
import subprocess
import sys
MIRRORS = ["", "https://gh-proxy.net/", "https://gh-proxy.com/"]
def pick_mirror(url: str):
"""HEAD 探测取 Content-Length;返回(实际可用 URL, 字节数)或 (None, 0)。"""
for m in MIRRORS:
u = m + url
r = subprocess.run(["curl", "-sIL", "-m", "30", u], capture_output=True, text=True)
size = 0
for line in r.stdout.splitlines():
if line.lower().startswith("content-length:"):
try:
size = int(line.split(":", 1)[1].strip())
except ValueError:
size = 0
if size > 0:
return u, size
return None, 0
def fetch(job) -> bool:
i, s, e, url, out = job
p = "%s.part%02d" % (out, i)
want = e - s + 1
for _attempt in range(200):
have = os.path.getsize(p) if os.path.exists(p) else 0
if have >= want:
return True
r0 = s + have
subprocess.run(
["curl", "-sL", "--speed-limit", "1024", "--speed-time", "30",
"-m", "600", "-r", "%d-%d" % (r0, e), "-o", p + ".new", url],
capture_output=True)
if os.path.exists(p + ".new"):
with open(p, "ab") as f, open(p + ".new", "rb") as g:
f.write(g.read())
os.remove(p + ".new")
return False
def main() -> int:
if len(sys.argv) < 3:
print("用法: python3 chunk_dl.py <URL> <输出路径> [分片数=16]")
return 1
url, out = sys.argv[1], sys.argv[2]
n = int(sys.argv[3]) if len(sys.argv) > 3 else 16
real, size = pick_mirror(url)
if not real:
print("全部下载源失败")
return 1
print("size=%d url=%s" % (size, real), flush=True)
chunk = (size + n - 1) // n
jobs = [(i, i * chunk, min((i + 1) * chunk, size) - 1, real, out)
for i in range(n) if i * chunk < size]
with cf.ThreadPoolExecutor(n) as ex:
for ok in ex.map(fetch, jobs):
if not ok:
print("分片重试耗尽")
return 1
with open(out, "wb") as f:
for i, _, _, _, _ in jobs:
with open("%s.part%02d" % (out, i), "rb") as g:
f.write(g.read())
for i, _, _, _, _ in jobs:
os.remove("%s.part%02d" % (out, i))
print("done %d" % os.path.getsize(out))
return 0
if __name__ == "__main__":
sys.exit(main())