From 70e0fd4d8b91c81a982aaed2ddde4d178675a017 Mon Sep 17 00:00:00 2001 From: Gio Della-Libera Date: Sun, 28 Jun 2026 07:53:32 -0700 Subject: [PATCH] Add hosted catalog config profiles (#95981) Merged via squash. Prepared head SHA: ef4d81cd0824de2322857028e64786a9dad304dd Co-authored-by: giodl73-repo <235387111+giodl73-repo@users.noreply.github.com> Co-authored-by: giodl73-repo <235387111+giodl73-repo@users.noreply.github.com> Reviewed-by: @giodl73-repo --- docs/.generated/config-baseline.sha256 | 6 +- scripts/plugin-sdk-surface-report.mjs | 6 +- src/config/schema.help.ts | 12 ++ src/config/schema.labels.ts | 6 + src/config/types.marketplaces.ts | 25 ++++ src/config/types.openclaw.ts | 3 + src/config/types.ts | 1 + src/config/zod-schema.marketplaces.test.ts | 131 ++++++++++++++++++ src/config/zod-schema.ts | 43 ++++++ .../official-external-plugin-catalog.test.ts | 43 ++++++ .../official-external-plugin-catalog.ts | 86 ++++++++---- 11 files changed, 330 insertions(+), 32 deletions(-) create mode 100644 src/config/types.marketplaces.ts create mode 100644 src/config/zod-schema.marketplaces.test.ts diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index 1d79002dcd74..52a4f634f005 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -f5a5855ddd7aa8c23a732f257eceaa20fd163b1d5f342c909f4aef15aa8643cf config-baseline.json -b8dffdb1a328aaf728a0707ab04d21c65f1a225a2360042e10832aa608699716 config-baseline.core.json +26afb40d889df462d5886e6fa0ecd09ce6bae12387836b59a2489c696a8ff3a1 config-baseline.json +00242f93938579d3ff3130f8d42e08a423e43dc7a73e3b5e5afb45b80c78d25a config-baseline.core.json 671979e86e4c4f59415d0a20879e838f9bbd883b3d29eeb02cb5131db8d187fe config-baseline.channel.json -94529978588d6e3776a86780b22cf9ff46a6f9957f2f178d3829403fad451ca7 config-baseline.plugin.json +21ce3d97ac3a83323fa29ddc35045c04f6eae3040c2c5515077f543b4608fd12 config-baseline.plugin.json diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index f610adee383f..ceacab5e8fb0 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -116,7 +116,7 @@ const defaultPublicDeprecatedExportsByEntrypointBudget = Object.freeze({ "approval-reply-runtime": 1, "config-runtime": 123, "config-contracts": 1, - "config-types": 416, + "config-types": 420, "config-schema": 3, "reply-dedupe": 1, "inbound-reply-dispatch": 33, @@ -202,11 +202,11 @@ let publicDeprecatedExportsByEntrypointBudget; try { budgets = { publicEntrypoints: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_ENTRYPOINTS", 322), - publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10388), + publicExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", 10392), publicFunctionExports: readBudgetEnv("OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", 5214), publicDeprecatedExports: readBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_DEPRECATED_EXPORTS", - 3249, + 3253, ), publicWildcardReexports: readBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_WILDCARD_REEXPORTS", diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 613d6832c4f3..f52dbacdd142 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -6,6 +6,18 @@ export const FIELD_HELP: Record = { meta: "Metadata fields automatically maintained by OpenClaw to record write/version history for this config file. Keep these values system-managed and avoid manual edits unless debugging migration history.", "meta.lastTouchedVersion": "Auto-set when OpenClaw writes the config.", "meta.lastTouchedAt": "ISO timestamp of the last config write (auto-set).", + marketplaces: + "Marketplace feed and local package source profile settings. Feeds provide package selection and governance metadata, while sources define the local source names that install candidates may reference.", + "marketplaces.feeds": + "Named marketplace feed profiles. The default public profile can be used as shipped, and deployments can add or override profiles to point OpenClaw at their effective feed endpoint.", + "marketplaces.feeds.*.url": + "HTTPS URL for the marketplace feed profile. Remote feed documents cannot introduce new registry domains or credentials; they only reference locally configured sources by name.", + "marketplaces.feeds.*.verification": + "Feed authenticity policy. This slice accepts only unsigned HTTPS feeds; signed verification is added when envelope enforcement is wired.", + "marketplaces.sources": + "Named package source profiles that feed entries can reference using sourceRef. Keep credentials and registry endpoints local so remote feeds cannot bootstrap trust roots.", + "marketplaces.sources.*.type": + "Package source profile type: npm, clawhub, or git. This slice validates sourceRef names only; registry and host endpoints are added when installer resolution can enforce them.", env: "Environment import and override settings used to supply runtime variables to the gateway process. Use this section to control shell-env loading and explicit variable injection behavior.", "env.shellEnv": "Shell environment import controls for loading variables from your login shell during startup. Keep this enabled when you depend on profile-defined secrets or PATH customizations.", diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts index 566c82fa46f1..87e36957b82e 100644 --- a/src/config/schema.labels.ts +++ b/src/config/schema.labels.ts @@ -5,6 +5,12 @@ export const FIELD_LABELS: Record = { meta: "Metadata", "meta.lastTouchedVersion": "Config Last Touched Version", "meta.lastTouchedAt": "Config Last Touched At", + marketplaces: "Marketplaces", + "marketplaces.feeds": "Marketplace Feeds", + "marketplaces.feeds.*.url": "Marketplace Feed URL", + "marketplaces.feeds.*.verification": "Marketplace Feed Verification", + "marketplaces.sources": "Marketplace Sources", + "marketplaces.sources.*.type": "Marketplace Source Type", env: "Environment", "env.shellEnv": "Shell Environment Import", "env.shellEnv.enabled": "Shell Environment Import Enabled", diff --git a/src/config/types.marketplaces.ts b/src/config/types.marketplaces.ts new file mode 100644 index 000000000000..3c5d2783f0e6 --- /dev/null +++ b/src/config/types.marketplaces.ts @@ -0,0 +1,25 @@ +// Defines marketplace feed and package source profile configuration types. +export type MarketplaceFeedVerificationConfig = { + mode: "unsigned"; +}; + +export type MarketplaceFeedProfileConfig = { + url: string; + verification?: MarketplaceFeedVerificationConfig; +}; + +export type MarketplaceSourceProfileConfig = + | { + type: "npm"; + } + | { + type: "clawhub"; + } + | { + type: "git"; + }; + +export type MarketplacesConfig = { + feeds?: Record; + sources?: Record; +}; diff --git a/src/config/types.openclaw.ts b/src/config/types.openclaw.ts index faf20921d247..3d838de1cabc 100644 --- a/src/config/types.openclaw.ts +++ b/src/config/types.openclaw.ts @@ -15,6 +15,7 @@ import type { CrestodianConfig } from "./types.crestodian.js"; import type { CronConfig } from "./types.cron.js"; import type { DiscoveryConfig, GatewayConfig, TalkConfig } from "./types.gateway.js"; import type { HooksConfig } from "./types.hooks.js"; +import type { MarketplacesConfig } from "./types.marketplaces.js"; import type { McpConfig } from "./types.mcp.js"; import type { MemoryConfig } from "./types.memory.js"; import type { @@ -176,6 +177,8 @@ export type OpenClawConfig = { }; /** Secret providers, defaults, and ref-resolution settings. */ secrets?: SecretsConfig; + /** Marketplace feed and local package source profile configuration. */ + marketplaces?: MarketplacesConfig; /** Skill loading and bundled skill configuration. */ skills?: SkillsConfig; /** Plugin registry/install/runtime configuration. */ diff --git a/src/config/types.ts b/src/config/types.ts index ff903d398e1e..a1f5aa01bf5b 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -8,6 +8,7 @@ export * from "./types.approvals.js"; export * from "./types.auth.js"; export * from "./types.base.js"; export * from "./types.browser.js"; +export * from "./types.marketplaces.js"; export * from "./types.channels.js"; export * from "./types.cli.js"; export * from "./types.commitments.js"; diff --git a/src/config/zod-schema.marketplaces.test.ts b/src/config/zod-schema.marketplaces.test.ts new file mode 100644 index 000000000000..b56cd7658272 --- /dev/null +++ b/src/config/zod-schema.marketplaces.test.ts @@ -0,0 +1,131 @@ +// Verifies marketplace feed and source profile config parsing. +import { describe, expect, it } from "vitest"; +import { OpenClawSchema } from "./zod-schema.js"; + +function expectMarketplacesConfig(value: unknown) { + const result = OpenClawSchema.safeParse(value); + if (!result.success) { + throw new Error(JSON.stringify(result.error.issues, null, 2)); + } + return result.data.marketplaces; +} + +describe("OpenClawSchema marketplaces config", () => { + it("accepts hosted feed and local source profiles", () => { + const marketplaces = expectMarketplacesConfig({ + marketplaces: { + feeds: { + "clawhub-public": { + url: "https://clawhub.ai/v1/feeds/plugins", + verification: { mode: "unsigned" }, + }, + acme: { + url: "https://packages.acme.example/openclaw/feed", + verification: { mode: "unsigned" }, + }, + }, + sources: { + "public-clawhub": { type: "clawhub" }, + "public-npm": { type: "npm" }, + "acme-npm": { type: "npm" }, + "acme-clawhub": { type: "clawhub" }, + "acme-git": { type: "git" }, + }, + }, + }); + + expect(marketplaces?.feeds?.acme.url).toBe("https://packages.acme.example/openclaw/feed"); + expect(marketplaces?.sources?.["acme-git"].type).toBe("git"); + }); + + it.each([ + "http://packages.acme.example/openclaw/feed", + "https://token@packages.acme.example/openclaw/feed", + "https://user:pass@packages.acme.example/openclaw/feed", + "https://packages.acme.example/openclaw/feed?token=secret", + "https://packages.acme.example/openclaw/feed#access-token", + "not a url", + ])("rejects invalid or auth-bearing hosted feed URL %s without throwing", (url) => { + expect(() => + OpenClawSchema.safeParse({ + marketplaces: { + feeds: { acme: { url } }, + }, + }), + ).not.toThrow(); + const result = OpenClawSchema.safeParse({ + marketplaces: { + feeds: { acme: { url } }, + }, + }); + + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues.map((issue) => issue.path.join("."))).toContain( + "marketplaces.feeds.acme.url", + ); + } + }); + + it("rejects refresh, auth, and signed verification until loader enforcement exists", () => { + expect( + OpenClawSchema.safeParse({ + marketplaces: { + feeds: { + acme: { + url: "https://packages.acme.example/openclaw/feed", + auth: { scheme: "bearer", secret: "token" }, + }, + }, + }, + }).success, + ).toBe(false); + expect( + OpenClawSchema.safeParse({ + marketplaces: { + feeds: { + acme: { + url: "https://packages.acme.example/openclaw/feed", + refresh: { onStartup: "if-stale" }, + }, + }, + }, + }).success, + ).toBe(false); + expect( + OpenClawSchema.safeParse({ + marketplaces: { + feeds: { + acme: { + url: "https://packages.acme.example/openclaw/feed", + verification: { mode: "signed" }, + }, + }, + }, + }).success, + ).toBe(false); + }); + + it("rejects unknown source profile types", () => { + const result = OpenClawSchema.safeParse({ + marketplaces: { + sources: { acme: { type: "container" } }, + }, + }); + + expect(result.success).toBe(false); + }); + + it("rejects source endpoints until installer resolution can enforce them", () => { + const result = OpenClawSchema.safeParse({ + marketplaces: { + sources: { + "acme-npm": { type: "npm", registry: "https://packages.acme.example/npm/" }, + "acme-clawhub": { type: "clawhub", baseUrl: "https://packages.acme.example/clawhub/" }, + }, + }, + }); + + expect(result.success).toBe(false); + }); +}); diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index b453048d1ff2..e358363a44a8 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -484,6 +484,48 @@ const CrestodianSchema = z .strict() .optional(); +function isPlainHttpsUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === "https:" && !url.username && !url.password && !url.search && !url.hash; + } catch { + return false; + } +} + +const MarketplaceVerificationSchema = z + .object({ + mode: z.literal("unsigned"), + }) + .strict(); + +const MarketplaceFeedProfileSchema = z + .object({ + url: z + .string() + .url() + .refine( + (value) => isPlainHttpsUrl(value), + "Expected https:// URL without credentials, query, or fragment", + ), + verification: MarketplaceVerificationSchema.optional(), + }) + .strict(); + +const MarketplaceSourceProfileSchema = z.union([ + z.object({ type: z.literal("npm") }).strict(), + z.object({ type: z.literal("clawhub") }).strict(), + z.object({ type: z.literal("git") }).strict(), +]); + +const MarketplacesSchema = z + .object({ + feeds: z.record(z.string().min(1), MarketplaceFeedProfileSchema).optional(), + sources: z.record(z.string().min(1), MarketplaceSourceProfileSchema).optional(), + }) + .strict() + .optional(); + const CommitmentsSchema = z .object({ enabled: z.boolean().optional(), @@ -739,6 +781,7 @@ export const OpenClawSchema = z .strict() .optional(), secrets: SecretsConfigSchema, + marketplaces: MarketplacesSchema, auth: z .object({ profiles: z diff --git a/src/plugins/official-external-plugin-catalog.test.ts b/src/plugins/official-external-plugin-catalog.test.ts index 88b7967d2223..6ced31fac0b0 100644 --- a/src/plugins/official-external-plugin-catalog.test.ts +++ b/src/plugins/official-external-plugin-catalog.test.ts @@ -13,6 +13,7 @@ import { isOfficialExternalPluginCatalogFeed, filterOfficialExternalPluginCatalogEntriesBySourceRefs, listOfficialExternalPluginCatalogEntries, + loadConfiguredHostedOfficialExternalPluginCatalogEntries, loadHostedOfficialExternalPluginCatalogEntries, parseOfficialExternalPluginCatalogEntries, resolveOfficialExternalProviderContractPluginIds, @@ -21,6 +22,7 @@ import { resolveOfficialExternalWebProviderContractPluginIdsForEnv, resolveOfficialExternalPluginId, resolveOfficialExternalPluginInstall, + resolveOfficialExternalPluginCatalogProfileConfigFromConfig, validateOfficialExternalPluginCatalogEntrySourceRefs, } from "./official-external-plugin-catalog.js"; @@ -464,6 +466,47 @@ describe("official external plugin catalog", () => { }); }); + it("loads hosted catalog profiles from OpenClaw config", async () => { + const config = { + marketplaces: { + feeds: { acme: { url: "https://packages.acme.example/openclaw/feed" } }, + sources: { "acme-npm": { type: "npm" as const } }, + }, + }; + const body = JSON.stringify({ + schemaVersion: 1, + id: "openclaw-official-external-plugins", + generatedAt: "2026-06-22T00:00:00.000Z", + sequence: 14, + entries: [ + { + name: "@acme/config-profile-proof", + kind: "plugin", + openclaw: { + plugin: { id: "config-profile-proof" }, + install: { sourceRef: "acme-npm", npmSpec: "@acme/config-profile-proof" }, + }, + }, + ], + }); + + expect(resolveOfficialExternalPluginCatalogProfileConfigFromConfig(config)).toBe( + config.marketplaces, + ); + + const result = await loadConfiguredHostedOfficialExternalPluginCatalogEntries(config, { + feedProfile: "acme", + fetchImpl: vi.fn(async (url: RequestInfo | URL) => { + expect(expectRequestUrl(url)).toBe("https://packages.acme.example/openclaw/feed"); + return new Response(body, { status: 200 }); + }), + snapshotStore: null, + }); + + expect(result.source).toBe("hosted"); + expect(result.entries.map((entry) => entry.name)).toEqual(["@acme/config-profile-proof"]); + }); + it("allows named local feed profiles to authorize their configured HTTPS host", async () => { const body = JSON.stringify({ schemaVersion: 1, diff --git a/src/plugins/official-external-plugin-catalog.ts b/src/plugins/official-external-plugin-catalog.ts index b2af12a19bab..e3245b98e16a 100644 --- a/src/plugins/official-external-plugin-catalog.ts +++ b/src/plugins/official-external-plugin-catalog.ts @@ -16,6 +16,16 @@ import type { type ManifestKey = typeof MANIFEST_KEY; +class HostedCatalogSnapshotWriteError extends Error { + readonly originalError: unknown; + + constructor(originalError: unknown) { + super("hosted catalog snapshot write failed"); + this.name = "HostedCatalogSnapshotWriteError"; + this.originalError = originalError; + } +} + export type OfficialExternalProviderAuthChoice = { method?: string; choiceId?: string; @@ -757,6 +767,8 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { ifNoneMatch?: string; ifModifiedSince?: string; expectedSha256?: string; + offline?: boolean; + requireSnapshotWrite?: boolean; snapshotStore?: HostedOfficialExternalPluginCatalogSnapshotStore | null; env?: NodeJS.ProcessEnv; stateDir?: string; @@ -780,10 +792,24 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { stateDir: params?.stateDir, stateDatabasePath: params?.stateDatabasePath, }); + const expectedSha256 = normalizeOptionalString(params?.expectedSha256); + const requireManifestInstallSourceRef = shouldRequireManifestInstallSourceRef({ + feedProfile: params?.feedProfile, + catalogConfig: params?.catalogConfig, + }); + if (params?.offline === true) { + return await snapshotOrBundledFallbackResult({ + error: "hosted catalog feed offline mode", + snapshotStore, + url: url.href, + expectedSha256, + catalogConfig: params?.catalogConfig, + requireManifestInstallSourceRef, + }); + } const headers = new Headers(); const ifNoneMatch = normalizeOptionalString(params?.ifNoneMatch); const ifModifiedSince = normalizeOptionalString(params?.ifModifiedSince); - const expectedSha256 = normalizeOptionalString(params?.expectedSha256); if (ifNoneMatch) { headers.set("if-none-match", ifNoneMatch); } @@ -827,10 +853,7 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { ifNoneMatch, ifModifiedSince, catalogConfig: params?.catalogConfig, - requireManifestInstallSourceRef: shouldRequireManifestInstallSourceRef({ - feedProfile: params?.feedProfile, - catalogConfig: params?.catalogConfig, - }), + requireManifestInstallSourceRef, }); } if (!response.ok) { @@ -843,10 +866,7 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { ifNoneMatch, ifModifiedSince, catalogConfig: params?.catalogConfig, - requireManifestInstallSourceRef: shouldRequireManifestInstallSourceRef({ - feedProfile: params?.feedProfile, - catalogConfig: params?.catalogConfig, - }), + requireManifestInstallSourceRef, }); } const body = await readHostedCatalogResponseText({ @@ -867,10 +887,7 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { ifNoneMatch, ifModifiedSince, catalogConfig: params?.catalogConfig, - requireManifestInstallSourceRef: shouldRequireManifestInstallSourceRef({ - feedProfile: params?.feedProfile, - catalogConfig: params?.catalogConfig, - }), + requireManifestInstallSourceRef, }); } const raw = JSON.parse(body) as unknown; @@ -884,20 +901,14 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { ifNoneMatch, ifModifiedSince, catalogConfig: params?.catalogConfig, - requireManifestInstallSourceRef: shouldRequireManifestInstallSourceRef({ - feedProfile: params?.feedProfile, - catalogConfig: params?.catalogConfig, - }), + requireManifestInstallSourceRef, }); } const entries = filterOfficialExternalPluginCatalogEntriesBySourceRefs( parseOfficialExternalPluginCatalogEntries(raw), { catalogConfig: params?.catalogConfig, - requireManifestInstallSourceRef: shouldRequireManifestInstallSourceRef({ - feedProfile: params?.feedProfile, - catalogConfig: params?.catalogConfig, - }), + requireManifestInstallSourceRef, }, ); await snapshotStore @@ -906,7 +917,11 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { metadata, savedAt: (params?.now?.() ?? new Date()).toISOString(), }) - .catch(() => undefined); + .catch((err: unknown) => { + if (params?.requireSnapshotWrite) { + throw new HostedCatalogSnapshotWriteError(err); + } + }); return { source: "hosted", entries: dedupeOfficialExternalPluginCatalogEntries(entries), @@ -914,6 +929,9 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { metadata, }; } catch (err) { + if (err instanceof HostedCatalogSnapshotWriteError) { + throw err.originalError; + } return await snapshotOrBundledFallbackResult({ error: err, snapshotStore, @@ -922,10 +940,7 @@ export async function loadHostedOfficialExternalPluginCatalogEntries(params?: { ifNoneMatch, ifModifiedSince, catalogConfig: params?.catalogConfig, - requireManifestInstallSourceRef: shouldRequireManifestInstallSourceRef({ - feedProfile: params?.feedProfile, - catalogConfig: params?.catalogConfig, - }), + requireManifestInstallSourceRef, }); } finally { if (response?.bodyUsed !== true) { @@ -1128,6 +1143,25 @@ export function resolveOfficialExternalPluginInstall( }; } +export function resolveOfficialExternalPluginCatalogProfileConfigFromConfig(config?: { + marketplaces?: OfficialExternalPluginCatalogProfileConfig; +}): OfficialExternalPluginCatalogProfileConfig | undefined { + return config?.marketplaces; +} + +export async function loadConfiguredHostedOfficialExternalPluginCatalogEntries( + config: { marketplaces?: OfficialExternalPluginCatalogProfileConfig } | undefined, + params?: Omit< + Parameters[0], + "catalogConfig" + >, +): Promise { + return await loadHostedOfficialExternalPluginCatalogEntries({ + ...params, + catalogConfig: resolveOfficialExternalPluginCatalogProfileConfigFromConfig(config), + }); +} + export function listOfficialExternalPluginCatalogEntries(): OfficialExternalPluginCatalogEntry[] { return dedupeOfficialExternalPluginCatalogEntries(bundledOfficialExternalPluginCatalogEntries()); }