工程(ci远端化·修复): mac_setup Godot URL模板修正(双stable+漏v前缀致404)+镜像假zip校验(200错误页体当失败, unzip -tq守卫双处)+chunk_dl.py收编(分片并行断点续传下载器,已实证方案仓库化)
This commit is contained in:
1 parent
f5c3fa935d
commit
fb5f2d42e5
2 files changed
+96
-4
No files matched your search
@@ -0,0 +1,88 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
r"""chunk_dl.py — 分片并行下载器(断点续传),2026-11-03 CI 迁移批收编。
|
||||||
|
|
||||||
|
背景:GitHub 直连在本项目两台机器均不可用,加速镜像站速度慢且会掐连接/回错误页。
|
||||||
|
本件把大文件切成 N 片经镜像并行拉取(单片断了自动续传重试),全部齐后拼合。
|
||||||
|
已实证:74MB Godot Linux 包约 15 分钟拉完(2026-11-03 mac_setup 依赖此件拉 Godot macOS 包)。
|
||||||
|
|
||||||
|
用法: python3 tools/ci/chunk_dl.py <URL> <输出路径> [分片数=16]
|
||||||
|
镜像回落顺序:直连 → gh-proxy.net → gh-proxy.com(HEAD 探测 Content-Length 决定实际用的源)。
|
||||||
|
校验:分片按字节范围落盘、按序拼合;zip 结构校验由调用方(mac_setup.sh 的 unzip -tq)承担。
|
||||||
|
退出码:0=成功 1=失败。
|
||||||
|
"""
|
||||||
|
import concurrent.futures as cf
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
MIRRORS = ["", "https://gh-proxy.net/", "https://gh-proxy.com/"]
|
||||||
|
|
||||||
|
|
||||||
|
def pick_mirror(url: str):
|
||||||
|
"""HEAD 探测取 Content-Length;返回(实际可用 URL, 字节数)或 (None, 0)。"""
|
||||||
|
for m in MIRRORS:
|
||||||
|
u = m + url
|
||||||
|
r = subprocess.run(["curl", "-sIL", "-m", "30", u], capture_output=True, text=True)
|
||||||
|
size = 0
|
||||||
|
for line in r.stdout.splitlines():
|
||||||
|
if line.lower().startswith("content-length:"):
|
||||||
|
try:
|
||||||
|
size = int(line.split(":", 1)[1].strip())
|
||||||
|
except ValueError:
|
||||||
|
size = 0
|
||||||
|
if size > 0:
|
||||||
|
return u, size
|
||||||
|
return None, 0
|
||||||
|
|
||||||
|
|
||||||
|
def fetch(job) -> bool:
|
||||||
|
i, s, e, url, out = job
|
||||||
|
p = "%s.part%02d" % (out, i)
|
||||||
|
want = e - s + 1
|
||||||
|
for _attempt in range(200):
|
||||||
|
have = os.path.getsize(p) if os.path.exists(p) else 0
|
||||||
|
if have >= want:
|
||||||
|
return True
|
||||||
|
r0 = s + have
|
||||||
|
subprocess.run(
|
||||||
|
["curl", "-sL", "--speed-limit", "1024", "--speed-time", "30",
|
||||||
|
"-m", "600", "-r", "%d-%d" % (r0, e), "-o", p + ".new", url],
|
||||||
|
capture_output=True)
|
||||||
|
if os.path.exists(p + ".new"):
|
||||||
|
with open(p, "ab") as f, open(p + ".new", "rb") as g:
|
||||||
|
f.write(g.read())
|
||||||
|
os.remove(p + ".new")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
if len(sys.argv) < 3:
|
||||||
|
print("用法: python3 chunk_dl.py <URL> <输出路径> [分片数=16]")
|
||||||
|
return 1
|
||||||
|
url, out = sys.argv[1], sys.argv[2]
|
||||||
|
n = int(sys.argv[3]) if len(sys.argv) > 3 else 16
|
||||||
|
real, size = pick_mirror(url)
|
||||||
|
if not real:
|
||||||
|
print("全部下载源失败")
|
||||||
|
return 1
|
||||||
|
print("size=%d url=%s" % (size, real), flush=True)
|
||||||
|
chunk = (size + n - 1) // n
|
||||||
|
jobs = [(i, i * chunk, min((i + 1) * chunk, size) - 1, real, out)
|
||||||
|
for i in range(n) if i * chunk < size]
|
||||||
|
with cf.ThreadPoolExecutor(n) as ex:
|
||||||
|
for ok in ex.map(fetch, jobs):
|
||||||
|
if not ok:
|
||||||
|
print("分片重试耗尽")
|
||||||
|
return 1
|
||||||
|
with open(out, "wb") as f:
|
||||||
|
for i, _, _, _, _ in jobs:
|
||||||
|
with open("%s.part%02d" % (out, i), "rb") as g:
|
||||||
|
f.write(g.read())
|
||||||
|
for i, _, _, _, _ in jobs:
|
||||||
|
os.remove("%s.part%02d" % (out, i))
|
||||||
|
print("done %d" % os.path.getsize(out))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -15,8 +15,8 @@
|
|||||||
# 幂等:重复执行只补缺件;账户密码首次生成后存 /var/db/gplan-ci-runner.pw(root 600),重跑复用。
|
# 幂等:重复执行只补缺件;账户密码首次生成后存 /var/db/gplan-ci-runner.pw(root 600),重跑复用。
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
GODOT_VER="4.7.2-stable"
|
GODOT_VER="4.7.2"
|
||||||
GODOT_URL="https://github.com/godotengine/godot/releases/download/${GODOT_VER}/Godot_${GODOT_VER}-stable_macos.universal.zip"
|
GODOT_URL="https://github.com/godotengine/godot/releases/download/${GODOT_VER}-stable/Godot_v${GODOT_VER}-stable_macos.universal.zip"
|
||||||
GDSTYLE_VER="0.4.0"
|
GDSTYLE_VER="0.4.0"
|
||||||
GDSTYLE_URL="https://github.com/atelico/gdstyle/releases/download/v${GDSTYLE_VER}/gdstyle-x86_64-apple-darwin.tar.gz"
|
GDSTYLE_URL="https://github.com/atelico/gdstyle/releases/download/v${GDSTYLE_VER}/gdstyle-x86_64-apple-darwin.tar.gz"
|
||||||
MIRRORS=("" "https://gh-proxy.net/" "https://gh-proxy.com/") # 直连优先,镜像回落
|
MIRRORS=("" "https://gh-proxy.net/" "https://gh-proxy.com/") # 直连优先,镜像回落
|
||||||
@@ -94,12 +94,16 @@ else
|
|||||||
_ok=0
|
_ok=0
|
||||||
for m in "${MIRRORS[@]}"; do
|
for m in "${MIRRORS[@]}"; do
|
||||||
log "下载 Godot(${m:+经 $m})"
|
log "下载 Godot(${m:+经 $m})"
|
||||||
if curl -fL --retry 2 -m 1800 -o "$ZIP" "${m}${GODOT_URL}"; then _ok=1; break; fi
|
# 镜像站可能回 200 的错误页体——下载后必须验 zip 结构,假成功当失败处理
|
||||||
|
if curl -fL --retry 2 -m 1800 -o "$ZIP" "${m}${GODOT_URL}" && unzip -tq "$ZIP" >/dev/null 2>&1; then
|
||||||
|
_ok=1; break
|
||||||
|
fi
|
||||||
log "该源失败,换下一源"
|
log "该源失败,换下一源"
|
||||||
done
|
done
|
||||||
[ "$_ok" = 1 ] || die "全部下载源失败——手动下载 ${GODOT_URL} 后重跑并加 --godot-zip <路径>"
|
[ "$_ok" = 1 ] || die "全部下载源失败——手动下载 ${GODOT_URL} 后重跑并加 --godot-zip <路径>;或用 tools/ci/chunk_dl.py 分片拉取"
|
||||||
fi
|
fi
|
||||||
TMPD="$(mktemp -d)"
|
TMPD="$(mktemp -d)"
|
||||||
|
unzip -tq "$ZIP" >/dev/null # 双保险:本地复用的 zip 同样验结构
|
||||||
unzip -q -o "$ZIP" -d "$TMPD"
|
unzip -q -o "$ZIP" -d "$TMPD"
|
||||||
install -m 0755 "${TMPD}/Godot.app/Contents/MacOS/Godot" /usr/local/bin/godot
|
install -m 0755 "${TMPD}/Godot.app/Contents/MacOS/Godot" /usr/local/bin/godot
|
||||||
xattr -dr com.apple.quarantine /usr/local/bin/godot 2>/dev/null || true
|
xattr -dr com.apple.quarantine /usr/local/bin/godot 2>/dev/null || true
|
||||||
|
|||||||
Reference in new issue
Block a user