97 lines
3.6 KiB
Python
97 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
||
r"""chunk_dl.py — 分片并行下载器（断点续传），2026-11-03 CI 迁移批收编。
|
||
|
||
背景：GitHub 直连在本项目两台机器均不可用，加速镜像站速度慢且会掐连接/回错误页。
|
||
本件把大文件切成 N 片经镜像并行拉取（单片断了自动续传重试），全部齐后拼合。
|
||
已实证：74MB Godot Linux 包约 15 分钟拉完（2026-11-03 mac_setup 依赖此件拉 Godot macOS 包）。
|
||
|
||
用法: python3 tools/ci/chunk_dl.py <URL> <输出路径> [分片数=16]
|
||
镜像回落顺序：直连 → gh-proxy.net → gh-proxy.com（HEAD 探测 Content-Length 决定实际用的源）。
|
||
校验：分片按字节范围落盘、按序拼合；zip 结构校验由调用方（mac_setup.sh 的 unzip -tq）承担。
|
||
退出码：0=成功 1=失败。
|
||
"""
|
||
import concurrent.futures as cf
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
|
||
MIRRORS = ["", "https://gh-proxy.net/", "https://gh-proxy.com/"]
|
||
|
||
|
||
def pick_mirror(url: str):
|
||
"""HEAD 探测取 Content-Length ＋ 前 4 字节 zip 魔数校验（镜像站会回 200 错误页体，
|
||
只看 Content-Length 会把 195 字节的假体当真——2026-11-03 gh-proxy.net 实证）。
|
||
返回（实际可用 URL, 字节数）或 (None, 0)。非 zip 目标禁用本件。"""
|
||
for m in MIRRORS:
|
||
u = m + url
|
||
r = subprocess.run(["curl", "-sIL", "-m", "30", u], capture_output=True, text=True)
|
||
size = 0
|
||
for line in r.stdout.splitlines():
|
||
if line.lower().startswith("content-length:"):
|
||
try:
|
||
size = int(line.split(":", 1)[1].strip())
|
||
except ValueError:
|
||
size = 0
|
||
if size <= 65536:
|
||
continue
|
||
head = subprocess.run(
|
||
["curl", "-sL", "-m", "20", "-r", "0-3", "-o", "-", u],
|
||
capture_output=True)
|
||
if head.stdout[:2] != b"PK":
|
||
continue
|
||
return u, size
|
||
return None, 0
|
||
|
||
|
||
def fetch(job) -> bool:
|
||
i, s, e, url, out = job
|
||
p = "%s.part%02d" % (out, i)
|
||
want = e - s + 1
|
||
for _attempt in range(200):
|
||
have = os.path.getsize(p) if os.path.exists(p) else 0
|
||
if have >= want:
|
||
return True
|
||
r0 = s + have
|
||
subprocess.run(
|
||
["curl", "-sL", "--speed-limit", "1024", "--speed-time", "30",
|
||
"-m", "600", "-r", "%d-%d" % (r0, e), "-o", p + ".new", url],
|
||
capture_output=True)
|
||
if os.path.exists(p + ".new"):
|
||
with open(p, "ab") as f, open(p + ".new", "rb") as g:
|
||
f.write(g.read())
|
||
os.remove(p + ".new")
|
||
return False
|
||
|
||
|
||
def main() -> int:
|
||
if len(sys.argv) < 3:
|
||
print("用法: python3 chunk_dl.py <URL> <输出路径> [分片数=16]")
|
||
return 1
|
||
url, out = sys.argv[1], sys.argv[2]
|
||
n = int(sys.argv[3]) if len(sys.argv) > 3 else 16
|
||
real, size = pick_mirror(url)
|
||
if not real:
|
||
print("全部下载源失败")
|
||
return 1
|
||
print("size=%d url=%s" % (size, real), flush=True)
|
||
chunk = (size + n - 1) // n
|
||
jobs = [(i, i * chunk, min((i + 1) * chunk, size) - 1, real, out)
|
||
for i in range(n) if i * chunk < size]
|
||
with cf.ThreadPoolExecutor(n) as ex:
|
||
for ok in ex.map(fetch, jobs):
|
||
if not ok:
|
||
print("分片重试耗尽")
|
||
return 1
|
||
with open(out, "wb") as f:
|
||
for i, _, _, _, _ in jobs:
|
||
with open("%s.part%02d" % (out, i), "rb") as g:
|
||
f.write(g.read())
|
||
for i, _, _, _, _ in jobs:
|
||
os.remove("%s.part%02d" % (out, i))
|
||
print("done %d" % os.path.getsize(out))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|