fix(env-sync): 同步 provider_config.json(UI 分组/顺序/模型级配置)
问题:只同步 v2/config.json 时,只在 v2/provider_config.json 里存在的 provider 分组到对面就是缺失的 —— 表现为「provider 数量对,但 UI 分组少几个」。 实测:9 个分组里有 4 个(GINKA API / zxcbug / zxcbug-p / d1)纯规则型, 密钥只存在 provider_config.json 的 config.access.apiKey 里,旧实现完全没收集。 修复: - sync-core:新增 V2_PROVIDER_CONFIG、extract/sanitize/collectProviderConfigSecrets、 providerConfigKeyManifest;密钥命名空间 PROVIDER_CONFIG/<providerId>/<key> - export:导出 v2.provider-config.json(脱敏),密钥并入加密束,manifest 记摘要与哈希 - import:按 providerId 合并规则、回填密钥空位、按 (providerId,modelId) 合并模型级配置、 providerOrder 以快照为准并保留本机特有分组;重复导入幂等 - import:provider_config 路径补「护住本机已有密钥 N 个」提示(原来只在 config.json 侧有) - 测试:单元 6 项 + E2E 2 项(分组完整同步/强制覆盖),全量 72 项通过
This commit is contained in:
@@ -7,13 +7,15 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import {
|
||||
HOME, USER, HOSTNAME, CLI_CONFIG, PLUGINS_HOME, REPO_DIR, V2_CONFIG,
|
||||
HOME, USER, HOSTNAME, CLI_CONFIG, PLUGINS_HOME, REPO_DIR, V2_CONFIG, V2_PROVIDER_CONFIG,
|
||||
nowTag, readJson, writeJson, encodeJsonPaths, encodePaths, fwd,
|
||||
officialPlugins, enabledPlugins, enabledPluginsDetailed, customPlugins, copyDirFiltered, hashDir,
|
||||
secretTargets, secretsManifest, buildSecretsBundle, encryptSecrets,
|
||||
requirePassphrase, scanForSecrets, ensureRepo, gitPush, dirSize, humanSize,
|
||||
collectModelSecrets, sanitizeProviders, summarizeProviders, modelSelection,
|
||||
extractModelSecrets, modelKeyManifest, WITH_MODELS, APPLY_MODEL_SELECTION,
|
||||
extractProviderConfigSecrets, collectProviderConfigSecrets, sanitizeProviderConfig,
|
||||
providerConfigKeyManifest, PROVIDER_CONFIG_PREFIX,
|
||||
} from "./sync-core.mjs";
|
||||
|
||||
function fail(e) { console.error(`导出失败:${e?.message ?? e}`); process.exitCode = 1; }
|
||||
@@ -69,6 +71,16 @@ try {
|
||||
const providerSanitized = v2cfg ? sanitizeProviders(v2cfg) : { providers: {}, withSecrets: [] };
|
||||
const modelKeys = v2cfg ? extractModelSecrets(v2cfg) : [];
|
||||
const modelKeysManifest = modelKeyManifest(modelKeys);
|
||||
/**
|
||||
* provider_config.json:UI 的分组/顺序/模型级配置。它自带一份 access.apiKey,
|
||||
* 与 config.json 那份可能重叠但不保证(只在 provider_config 里存在的分组,
|
||||
* 其密钥也只在它这)。两份都要收,否则那些分组过去就是个空壳。
|
||||
*/
|
||||
const pcCfg = WITH_MODELS ? readJson(V2_PROVIDER_CONFIG, null) : null;
|
||||
const pcSanitized = pcCfg ? sanitizeProviderConfig(pcCfg) : { config: null, withSecrets: [] };
|
||||
const pcKeys = pcCfg ? extractProviderConfigSecrets(pcCfg) : [];
|
||||
const pcKeysManifest = providerConfigKeyManifest(pcKeys);
|
||||
const pcSecrets = pcCfg ? collectProviderConfigSecrets(pcCfg) : [];
|
||||
let modelKeyCount = 0;
|
||||
if (v2cfg) {
|
||||
// 脱敏结果里不允许再有真密钥(helper 漏了就直接停,别把明文推上去)
|
||||
@@ -82,6 +94,17 @@ try {
|
||||
modelKeyCount++;
|
||||
}
|
||||
}
|
||||
if (pcCfg) {
|
||||
const rules = pcSanitized.config?.config?.providerConfigRules?.providerRules ?? [];
|
||||
const left = rules.flatMap((r) => scanForSecrets(r?.config?.access ?? {}));
|
||||
if (left.length) throw new Error(`provider_config 脱敏不彻底(${left.slice(0, 5).join(",")}),已中断`);
|
||||
for (const { provider, key, value } of pcKeys) {
|
||||
bundle.files.push({
|
||||
rel: `${PROVIDER_CONFIG_PREFIX}${provider}/${key}`,
|
||||
contentB64: Buffer.from(String(value), "utf8").toString("base64"),
|
||||
});
|
||||
}
|
||||
}
|
||||
const enc = encryptSecrets(bundle, pass);
|
||||
|
||||
const tag = `${HOSTNAME}-${nowTag()}`;
|
||||
@@ -112,10 +135,22 @@ try {
|
||||
modelKeyCount,
|
||||
});
|
||||
}
|
||||
if (pcCfg) {
|
||||
Object.assign(manifest, {
|
||||
// UI 分组/顺序/模型级配置的摘要(结构本体在 v2.provider-config.json)
|
||||
providerConfigOrder: pcSanitized.config?.config?.providerOrder ?? [],
|
||||
providerConfigRuleCount: (pcSanitized.config?.config?.providerConfigRules?.providerRules ?? []).length,
|
||||
providerConfigModelRuleCount: (pcSanitized.config?.config?.modelConfigRules?.providerModelRules ?? []).length,
|
||||
providerConfigWithSecrets: pcSanitized.withSecrets,
|
||||
providerConfigKeys: pcKeysManifest,
|
||||
providerConfigKeyCount: pcKeys.length,
|
||||
});
|
||||
}
|
||||
// manifest 与 latest.json 都留到自检通过后再落盘:失败时不留半个快照
|
||||
writeJson(path.join(snapDir, "mcp.servers.json"), mcpEncoded);
|
||||
writeJson(path.join(snapDir, "plugins-dirs.json"), dirsEncoded);
|
||||
if (v2cfg) writeJson(path.join(snapDir, "v2.providers.json"), encodeJsonPaths(providerSanitized.providers));
|
||||
if (pcCfg) writeJson(path.join(snapDir, "v2.provider-config.json"), encodeJsonPaths(pcSanitized.config));
|
||||
fs.writeFileSync(path.join(snapDir, "secrets.enc"), enc);
|
||||
writeJson(path.join(snapDir, "secrets.manifest.json"), manifest.secretFiles);
|
||||
|
||||
@@ -138,7 +173,7 @@ try {
|
||||
|
||||
// 密钥:全覆盖,命中即硬失败
|
||||
const plainText = readAll(snapFiles) + "\n" + pendingManifestText;
|
||||
const leakedSecrets = [...providerSecrets, ...secrets.map((s) => {
|
||||
const leakedSecrets = [...providerSecrets, ...pcSecrets, ...secrets.map((s) => {
|
||||
try { return fs.readFileSync(s.abs, "utf8").trim(); } catch { return ""; }
|
||||
})].filter((v) => v && v.length >= 8 && plainText.includes(v));
|
||||
if (leakedSecrets.length) {
|
||||
@@ -214,7 +249,7 @@ try {
|
||||
});
|
||||
// 只提交本次快照 + latest.json:不做 add -A,免得把别的会话未推送的快照连带推上去
|
||||
const pushMsg = gitPush(
|
||||
`sync-export ${tag} (official:${official.length} custom:${custom.length} mcp:${Object.keys(mcpServers).length} secrets:${secrets.length}${v2cfg ? ` providers:${providerSummary.length}` : ""})`,
|
||||
`sync-export ${tag} (official:${official.length} custom:${custom.length} mcp:${Object.keys(mcpServers).length} secrets:${secrets.length}${v2cfg ? ` providers:${providerSummary.length}` : ""}${pcCfg ? ` groups:${(pcSanitized.config?.config?.providerOrder ?? []).length}` : ""})`,
|
||||
{ paths: [path.posix.join("snapshots", tag), "latest.json", ".gitattributes", ".gitignore", "local-overrides.example.json"] });
|
||||
|
||||
console.log(`# 导出成功`);
|
||||
@@ -227,6 +262,12 @@ try {
|
||||
console.log(`- 模型 provider:${providerSummary.length} 个(脱敏结构明文),密钥 ${modelKeyCount} 个已入加密束`);
|
||||
console.log(`- 模型选中态:${APPLY_MODEL_SELECTION ? "已随快照应用" : "仅记录,导入时不覆盖本机"}`);
|
||||
}
|
||||
if (pcCfg) {
|
||||
const order = pcSanitized.config?.config?.providerOrder ?? [];
|
||||
const rules = pcSanitized.config?.config?.providerConfigRules?.providerRules ?? [];
|
||||
const mRules = pcSanitized.config?.config?.modelConfigRules?.providerModelRules ?? [];
|
||||
console.log(`- UI 分组:${order.length} 个(providerOrder),规则 ${rules.length} 条,模型级配置 ${mRules.length} 条,密钥 ${pcKeys.length} 个已入加密束`);
|
||||
}
|
||||
console.log(`- 明文区自检:无密钥泄漏、配置区无本机路径残留`);
|
||||
console.log(`- 快照体积:${humanSize(manifest.bytes)}`);
|
||||
if (pathWarnings.length) {
|
||||
|
||||
@@ -10,12 +10,13 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
HOME, CLI_CONFIG, PLUGINS_HOME, REPO_DIR, BACKUP_ROOT, V2_CONFIG, V2_SETTING,
|
||||
HOME, CLI_CONFIG, PLUGINS_HOME, REPO_DIR, BACKUP_ROOT, V2_CONFIG, V2_SETTING, V2_PROVIDER_CONFIG,
|
||||
WITH_MODELS, APPLY_MODEL_SELECTION, OVERWRITE_MODEL_SECRETS, MODEL_SECRET_REF,
|
||||
nowTag, readJson, writeJson, decodeJsonPaths, encodeJsonPaths,
|
||||
decryptSecrets, restoreRelPath, isModelEntry, parseModelRel, requirePassphrase,
|
||||
ensureRepo, readSnapshotManifest, latestSnapshotDir,
|
||||
deepMerge, loadOverrides,
|
||||
isProviderConfigEntry, parseProviderConfigRel,
|
||||
} from "./sync-core.mjs";
|
||||
|
||||
function fail(e) { console.error(`导入失败:${e?.message ?? e}`); process.exitCode = 1; }
|
||||
@@ -101,9 +102,11 @@ try {
|
||||
const skipSet = skipSecretsSet();
|
||||
const skipped = [];
|
||||
const modelKeys = [];
|
||||
const pcKeys = [];
|
||||
let n = 0;
|
||||
for (const f of bundle.files ?? []) {
|
||||
if (isModelEntry(f.rel)) { modelKeys.push(f); continue; }
|
||||
if (isProviderConfigEntry(f.rel)) { pcKeys.push(f); continue; }
|
||||
if (skipSet.has(f.rel)) { skipped.push(f.rel); continue; }
|
||||
let dst;
|
||||
try { dst = restoreRelPath(f.rel); }
|
||||
@@ -190,6 +193,94 @@ try {
|
||||
report.push(`- 模型:快照未含模型(导出时未开 ZCODE_SYNC_WITH_MODELS=1)`);
|
||||
}
|
||||
|
||||
// 4b) provider_config.json:UI 的分组/顺序/模型级配置
|
||||
// config.json 决定「provider 能不能调」,这份决定「UI 怎么分组显示」。
|
||||
// 只同步前者时,仅在 provider_config 里存在的分组到对面就是缺失的。
|
||||
const pcFile = path.join(snapDir, "v2.provider-config.json");
|
||||
if (fs.existsSync(pcFile) && (manifest.providerConfigRuleCount ?? 0) > 0) {
|
||||
const snapPc = decodeJsonPaths(readJson(pcFile, {}));
|
||||
if (!WITH_MODELS) {
|
||||
report.push(`- UI 分组:快照含 ${(manifest.providerConfigOrder ?? []).length} 个分组,`
|
||||
+ `默认不覆盖本机(设 ZCODE_SYNC_WITH_MODELS=1 应用)`);
|
||||
} else {
|
||||
const localPc = fs.existsSync(V2_PROVIDER_CONFIG) ? readJson(V2_PROVIDER_CONFIG, {}) : {};
|
||||
const b3 = backup(V2_PROVIDER_CONFIG); if (b3) report.push(`- 备份 v2/provider_config.json → ${b3}`);
|
||||
const keyMapPc = new Map();
|
||||
for (const f of pcKeys) {
|
||||
const { provider, key } = parseProviderConfigRel(f.rel);
|
||||
keyMapPc.set(`${provider}\n${key}`, Buffer.from(f.contentB64, "base64").toString("utf8"));
|
||||
}
|
||||
const snapCfg = snapPc?.config ?? {};
|
||||
const localCfg = localPc.config ?? (localPc.config = {});
|
||||
const snapRules = snapCfg.providerConfigRules?.providerRules ?? [];
|
||||
const snapModels = snapCfg.modelConfigRules?.providerModelRules ?? [];
|
||||
const snapManual = snapCfg.modelConfigRules?.manualProviderModelRules ?? [];
|
||||
localCfg.providerConfigRules = localCfg.providerConfigRules ?? {};
|
||||
localCfg.modelConfigRules = localCfg.modelConfigRules ?? {};
|
||||
const localRules = localCfg.providerConfigRules.providerRules ?? (localCfg.providerConfigRules.providerRules = []);
|
||||
const localModels = localCfg.modelConfigRules.providerModelRules ?? (localCfg.modelConfigRules.providerModelRules = []);
|
||||
const localManual = localCfg.modelConfigRules.manualProviderModelRules ?? (localCfg.modelConfigRules.manualProviderModelRules = []);
|
||||
// 按 id 建索引,避免同一 provider 重复追加
|
||||
const byId = new Map(localRules.map((r, i) => [r?.providerId, i]));
|
||||
let addRule = 0, mergeRule = 0, filledPcKey = 0, skippedPcKey = 0, skippedPcOverwrite = 0, addModel = 0;
|
||||
for (const sr of snapRules) {
|
||||
const id = sr?.providerId;
|
||||
if (!id) continue;
|
||||
if (!byId.has(id)) { localRules.push(JSON.parse(JSON.stringify(sr))); byId.set(id, localRules.length - 1); addRule++; }
|
||||
else {
|
||||
// 已有:补缺失字段,不动本机已有的值(含密钥)
|
||||
const lr = localRules[byId.get(id)];
|
||||
lr.providerName = lr.providerName ?? sr.providerName;
|
||||
lr.config = lr.config ?? {};
|
||||
for (const [k, v] of Object.entries(sr.config ?? {})) {
|
||||
if (k === "access") continue; // 密钥单独处理
|
||||
if (lr.config[k] === undefined) lr.config[k] = JSON.parse(JSON.stringify(v));
|
||||
}
|
||||
lr.config.access = lr.config.access ?? {};
|
||||
for (const [k, v] of Object.entries(sr.config?.access ?? {})) {
|
||||
if (v === MODEL_SECRET_REF) continue;
|
||||
if (lr.config.access[k] === undefined) lr.config.access[k] = v;
|
||||
}
|
||||
mergeRule++;
|
||||
}
|
||||
// 密钥回填:占位符/空/过短才填
|
||||
const lr = localRules[byId.get(id)];
|
||||
lr.config.access = lr.config.access ?? {};
|
||||
for (const [kk, vv] of keyMapPc) {
|
||||
const [pid, k] = kk.split("\n");
|
||||
if (pid !== id) continue;
|
||||
if (skipSet.has(`PROVIDER_CONFIG/${pid}/${k}`)) { skippedPcKey++; continue; }
|
||||
const cur = lr.config.access[k];
|
||||
if (cur === MODEL_SECRET_REF || cur === undefined || cur === "" || (typeof cur === "string" && cur.length < 8)) {
|
||||
lr.config.access[k] = vv; filledPcKey++;
|
||||
} else if (OVERWRITE_MODEL_SECRETS) {
|
||||
lr.config.access[k] = vv; filledPcKey++;
|
||||
} else {
|
||||
skippedPcOverwrite++; // 本机已有非空密钥,默认护住
|
||||
}
|
||||
}
|
||||
}
|
||||
// 模型级配置:按 (providerId, modelId) 去重合并
|
||||
const mKey = (r) => `${r?.providerId}\n${r?.modelId}`;
|
||||
const haveModel = new Set(localModels.map(mKey));
|
||||
for (const mr of snapModels) { if (!haveModel.has(mKey(mr))) { localModels.push(JSON.parse(JSON.stringify(mr))); haveModel.add(mKey(mr)); addModel++; } }
|
||||
const haveManual = new Set(localManual.map(mKey));
|
||||
for (const mr of snapManual) { if (!haveManual.has(mKey(mr))) { localManual.push(JSON.parse(JSON.stringify(mr))); haveManual.add(mKey(mr)); } }
|
||||
// providerOrder:快照的分组顺序(合并本机多出来的,不让本机特有分组消失)
|
||||
const snapOrder = snapCfg.providerOrder ?? [];
|
||||
const merged = [...snapOrder];
|
||||
for (const id of localCfg.providerOrder ?? []) if (!merged.includes(id)) merged.push(id);
|
||||
localCfg.providerOrder = merged;
|
||||
if (localPc.schemaVersion === undefined && snapPc.schemaVersion !== undefined) localPc.schemaVersion = snapPc.schemaVersion;
|
||||
writeJson(V2_PROVIDER_CONFIG, localPc);
|
||||
report.push(`- UI 分组:${merged.length} 个分组(新增规则 ${addRule},合并 ${mergeRule},密钥回填 ${filledPcKey}`
|
||||
+ (skippedPcOverwrite ? `,护住本机已有密钥 ${skippedPcOverwrite} 个(需覆盖设 ZCODE_SYNC_OVERWRITE_MODEL_SECRETS=1)` : "")
|
||||
+ (skippedPcKey ? `,跳过 ${skippedPcKey}(ZCODE_SYNC_SKIP_SECRETS)` : "") + `),模型级配置 +${addModel} 条`);
|
||||
}
|
||||
} else if (manifest.providerConfigRuleCount) {
|
||||
report.push(`- UI 分组:快照仅记录摘要 ${manifest.providerConfigRuleCount} 条(未含配置本体)`);
|
||||
}
|
||||
|
||||
// 5) 官方插件:只报告差异,不自动执行(去插件市场对照启用,重启生效,避免误装)
|
||||
if (process.env.ZCODE_SYNC_APPLY_OFFICIAL !== "0") {
|
||||
const { enabledPlugins } = await import("./sync-core.mjs");
|
||||
|
||||
@@ -25,6 +25,8 @@ export const REPO_DIR = path.join(WORK, "repo");
|
||||
export const BACKUP_ROOT = path.join(WORK, "backup");
|
||||
export const V2_CONFIG = path.join(ZCODE_HOME, "v2", "config.json");
|
||||
export const V2_SETTING = path.join(ZCODE_HOME, "v2", "setting.json");
|
||||
/** UI 的分组/排序/模型级配置(providerOrder、group、personalModelIds、contextWindow 等) */
|
||||
export const V2_PROVIDER_CONFIG = path.join(ZCODE_HOME, "v2", "provider_config.json");
|
||||
/** 模型同步默认关闭,显式开启才同步(ZCODE_SYNC_WITH_MODELS=1) */
|
||||
export const WITH_MODELS = process.env.ZCODE_SYNC_WITH_MODELS === "1";
|
||||
/** 模型选中态(当前用哪个provider/model)默认不覆盖本机,显式开启才应用 */
|
||||
@@ -318,6 +320,69 @@ export function parseModelRel(rel) {
|
||||
return { provider: rest.slice(0, i), key: rest.slice(i + 1) };
|
||||
}
|
||||
|
||||
/* ---------- provider_config.json:UI 分组/顺序/模型级配置 ----------
|
||||
* config.json 只管「provider 能不能调」;UI 上看到的 9 个分组、拖拽顺序、
|
||||
* 每条 provider 属于哪个 group、模型级参数(contextWindow 等)全在 provider_config.json。
|
||||
* 早期版本只同步 config.json,于是只在 provider_config 里存在的分组整个丢失。
|
||||
*/
|
||||
export const PROVIDER_CONFIG_PREFIX = "PROVIDER_CONFIG/";
|
||||
export function isProviderConfigEntry(rel) { return String(rel).startsWith(PROVIDER_CONFIG_PREFIX); }
|
||||
/** PROVIDER_CONFIG/<providerId>/<key> → { provider, key } */
|
||||
export function parseProviderConfigRel(rel) {
|
||||
const rest = String(rel).slice(PROVIDER_CONFIG_PREFIX.length);
|
||||
const i = rest.indexOf("/");
|
||||
if (i <= 0 || i === rest.length - 1) throw new Error(`非法 provider_config 密钥路径:${rel}`);
|
||||
return { provider: rest.slice(0, i), key: rest.slice(i + 1) };
|
||||
}
|
||||
/** providerRules 里的真密钥(内存中处理,调用方不得打印) */
|
||||
export function extractProviderConfigSecrets(pcfg) {
|
||||
const out = [];
|
||||
for (const r of pcfg?.config?.providerConfigRules?.providerRules ?? []) {
|
||||
const id = r?.providerId;
|
||||
if (!id) continue;
|
||||
for (const [k, v] of Object.entries(r?.config?.access ?? {})) {
|
||||
if (typeof v === "string" && MODEL_KEY_RE.test(k) && v.length >= 8) out.push({ provider: id, key: k, value: v });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
/** 全量收集,供泄漏扫描比对 */
|
||||
export function collectProviderConfigSecrets(pcfg) {
|
||||
const vals = new Set();
|
||||
const walk = (node) => {
|
||||
if (Array.isArray(node)) return node.forEach(walk);
|
||||
if (node && typeof node === "object") {
|
||||
for (const [k, v] of Object.entries(node)) {
|
||||
if (typeof v === "string" && MODEL_KEY_RE.test(k) && v.length >= 8) vals.add(v);
|
||||
else walk(v);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(pcfg?.config?.providerConfigRules?.providerRules ?? []);
|
||||
return [...vals];
|
||||
}
|
||||
/** 脱敏:providerRules 里的密钥换成占位符;返回哪些 provider 带密钥(只记有无) */
|
||||
export function sanitizeProviderConfig(pcfg) {
|
||||
const clean = JSON.parse(JSON.stringify(pcfg ?? {}));
|
||||
const withSecrets = [];
|
||||
for (const r of clean?.config?.providerConfigRules?.providerRules ?? []) {
|
||||
const id = r?.providerId;
|
||||
let hit = false;
|
||||
for (const [k, v] of Object.entries(r?.config?.access ?? {})) {
|
||||
if (typeof v === "string" && MODEL_KEY_RE.test(k) && v.length >= 8) { r.config.access[k] = MODEL_SECRET_REF; hit = true; }
|
||||
}
|
||||
if (hit && id) withSecrets.push(id);
|
||||
}
|
||||
return { config: clean, withSecrets };
|
||||
}
|
||||
/** provider_config 密钥清单(只记 provider/key/sha) */
|
||||
export function providerConfigKeyManifest(keys) {
|
||||
return (keys ?? []).map(({ provider, key, value }) => ({
|
||||
rel: `${PROVIDER_CONFIG_PREFIX}${provider}/${key}`,
|
||||
sha256: crypto.createHash("sha256").update(String(value), "utf8").digest("hex"),
|
||||
}));
|
||||
}
|
||||
|
||||
/* ---------- 脱敏扫描:明文区不得出现密钥值 ---------- */
|
||||
/** 递归收集 v2 provider 里的敏感值(内存中比对,绝不打印) */
|
||||
export function collectModelSecrets(v2cfg) {
|
||||
|
||||
Reference in New Issue
Block a user