Give repo Faces open-web Eyes
This commit is contained in:
parent
da6cd83ae1
commit
db62ed7f77
5 changed files with 268 additions and 1 deletions
|
|
@ -58,6 +58,11 @@ export const searchSourcesInputSchema = {
|
|||
language: z.string().min(1).optional(),
|
||||
};
|
||||
|
||||
export const searchWebInputSchema = {
|
||||
query: z.string().min(1).max(240),
|
||||
limit: z.number().int().min(1).max(8).optional(),
|
||||
};
|
||||
|
||||
export const sourceContextInputSchema = {
|
||||
sourceId: z.string().min(1),
|
||||
chunkIndex: z.number().int().min(0).optional(),
|
||||
|
|
@ -123,6 +128,11 @@ export interface SearchSourcesArgs {
|
|||
language?: string;
|
||||
}
|
||||
|
||||
export interface SearchWebArgs {
|
||||
query: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface SourceContextArgs {
|
||||
sourceId: string;
|
||||
chunkIndex?: number;
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import {
|
|||
type RuntimeInfoArgs,
|
||||
type SearchHistoryArgs,
|
||||
type SearchSourcesArgs,
|
||||
type SearchWebArgs,
|
||||
type SourceContextArgs,
|
||||
formatArchivedMessage,
|
||||
formatHistoryResults,
|
||||
|
|
@ -46,6 +47,7 @@ import {
|
|||
runtimeInfoInputSchema,
|
||||
searchHistoryInputSchema,
|
||||
searchSourcesInputSchema,
|
||||
searchWebInputSchema,
|
||||
sourceContextInputSchema,
|
||||
} from "./mcp-server-shared";
|
||||
|
||||
|
|
@ -56,6 +58,13 @@ const READ_ONLY_ANNOTATIONS = {
|
|||
openWorldHint: false,
|
||||
} as const;
|
||||
|
||||
const OPEN_WORLD_READ_ONLY_ANNOTATIONS = {
|
||||
readOnlyHint: true,
|
||||
destructiveHint: false,
|
||||
idempotentHint: false,
|
||||
openWorldHint: true,
|
||||
} as const;
|
||||
|
||||
function isMcpToolAllowed(name: string): boolean {
|
||||
const raw = process.env.VOIDBOT_MCP_TOOL_ALLOWLIST?.trim();
|
||||
if (!raw) {
|
||||
|
|
@ -192,6 +201,41 @@ export function registerVoidbotTools(
|
|||
},
|
||||
);
|
||||
|
||||
registerIfAllowed(
|
||||
"search_web",
|
||||
{
|
||||
title: "Search The Web",
|
||||
description:
|
||||
"Search the public internet for fresh outside references, examples, cultural material, technical terms, and unfamiliar concepts. Use this as the Face's local Eyes when repo/archive memory feels too narrow. Treat results as leads, not durable truth.",
|
||||
inputSchema: searchWebInputSchema,
|
||||
annotations: OPEN_WORLD_READ_ONLY_ANNOTATIONS,
|
||||
},
|
||||
async (input: SearchWebArgs): Promise<CallToolResult> => {
|
||||
const results = await searchWeb(input.query, input.limit ?? 5);
|
||||
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
results.length > 0
|
||||
? renderJsonBlock({
|
||||
query: input.query,
|
||||
resultCount: results.length,
|
||||
results,
|
||||
})
|
||||
: `No public web results were found for "${input.query}".`,
|
||||
},
|
||||
],
|
||||
structuredContent: {
|
||||
query: input.query,
|
||||
resultCount: results.length,
|
||||
results,
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
registerIfAllowed(
|
||||
"search_sources",
|
||||
{
|
||||
|
|
@ -797,6 +841,216 @@ export function registerVoidbotTools(
|
|||
);
|
||||
}
|
||||
|
||||
async function searchWeb(
|
||||
query: string,
|
||||
limit: number,
|
||||
): Promise<Array<{ title: string; url: string; snippet?: string }>> {
|
||||
const requestedLimit = Math.max(1, Math.min(8, Math.trunc(limit)));
|
||||
try {
|
||||
const mojeekResults = parseMojeekResults(await fetchMojeekHtml(query), requestedLimit);
|
||||
if (mojeekResults.length > 0) {
|
||||
return mojeekResults;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to other public search surfaces.
|
||||
}
|
||||
|
||||
try {
|
||||
const duckDuckGoResults = parseDuckDuckGoResults(await fetchDuckDuckGoHtml(query), requestedLimit);
|
||||
if (duckDuckGoResults.length > 0) {
|
||||
return duckDuckGoResults;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to Bing before reporting failure.
|
||||
}
|
||||
|
||||
const bingUrl = new URL("https://www.bing.com/search");
|
||||
bingUrl.searchParams.set("q", query);
|
||||
|
||||
const bingResponse = await fetch(bingUrl, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; VoidBot Face Eyes/0.1; +https://github.com/GameCult/VoidBot)",
|
||||
Accept: "text/html,application/xhtml+xml",
|
||||
},
|
||||
});
|
||||
|
||||
if (!bingResponse.ok) {
|
||||
throw new Error(`Web search failed: ${bingResponse.status} ${bingResponse.statusText}`);
|
||||
}
|
||||
|
||||
return parseBingResults(await bingResponse.text(), requestedLimit);
|
||||
}
|
||||
|
||||
async function fetchMojeekHtml(query: string): Promise<string> {
|
||||
const url = new URL("https://www.mojeek.com/search");
|
||||
url.searchParams.set("q", query);
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; VoidBot Face Eyes/0.1; +https://github.com/GameCult/VoidBot)",
|
||||
Accept: "text/html,application/xhtml+xml",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Web search failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.text();
|
||||
}
|
||||
|
||||
function parseMojeekResults(
|
||||
html: string,
|
||||
limit: number,
|
||||
): Array<{ title: string; url: string; snippet?: string }> {
|
||||
const results: Array<{ title: string; url: string; snippet?: string }> = [];
|
||||
const resultPattern = /<!--rs--><li[^>]*>[\s\S]*?<h2>\s*<a[^>]+class="title"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>\s*<\/h2>[\s\S]*?<p class="s">([\s\S]*?)<\/p>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = resultPattern.exec(html)) && results.length < limit) {
|
||||
const rawUrl = decodeHtml(match[1] ?? "");
|
||||
const title = normalizeWhitespace(stripHtml(match[2] ?? ""));
|
||||
const snippet = normalizeWhitespace(stripHtml(match[3] ?? ""));
|
||||
if (title && rawUrl) {
|
||||
results.push({
|
||||
title,
|
||||
url: rawUrl,
|
||||
...(snippet ? { snippet } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function parseBingResults(
|
||||
html: string,
|
||||
limit: number,
|
||||
): Array<{ title: string; url: string; snippet?: string }> {
|
||||
const results: Array<{ title: string; url: string; snippet?: string }> = [];
|
||||
const resultPattern = /<li class="b_algo"[\s\S]*?<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>\s*<\/h2>([\s\S]*?)(?=<li class="b_algo"|<li class="b_ans"|<div id="b_pag"|<\/ol>)/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = resultPattern.exec(html)) && results.length < limit) {
|
||||
const rawUrl = decodeHtml(match[1] ?? "");
|
||||
const title = normalizeWhitespace(stripHtml(match[2] ?? ""));
|
||||
const body = match[3] ?? "";
|
||||
const snippetMatch = body.match(/<p[^>]*>([\s\S]*?)<\/p>/i);
|
||||
const snippet = snippetMatch ? normalizeWhitespace(stripHtml(snippetMatch[1] ?? "")) : undefined;
|
||||
const resolvedUrl = normalizeBingResultUrl(rawUrl);
|
||||
if (!title || !resolvedUrl) {
|
||||
continue;
|
||||
}
|
||||
results.push({
|
||||
title,
|
||||
url: resolvedUrl,
|
||||
...(snippet ? { snippet } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function normalizeBingResultUrl(rawUrl: string): string | undefined {
|
||||
try {
|
||||
const decodedRawUrl = decodeHtml(rawUrl);
|
||||
const parsed = new URL(decodedRawUrl);
|
||||
const encodedTarget = parsed.searchParams.get("u")
|
||||
?? decodedRawUrl.match(/[?&]u=([^&]+)/)?.[1];
|
||||
if (!encodedTarget) {
|
||||
return parsed.toString();
|
||||
}
|
||||
const normalized = decodeURIComponent(encodedTarget).startsWith("a1")
|
||||
? decodeURIComponent(encodedTarget).slice(2)
|
||||
: decodeURIComponent(encodedTarget);
|
||||
return decodeBase64Url(normalized);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function decodeBase64Url(value: string): string {
|
||||
const normalized = value.replace(/-/g, "+").replace(/_/g, "/");
|
||||
const padded = normalized.padEnd(Math.ceil(normalized.length / 4) * 4, "=");
|
||||
return Buffer.from(padded, "base64").toString("utf8");
|
||||
}
|
||||
|
||||
async function fetchDuckDuckGoHtml(query: string): Promise<string> {
|
||||
const url = new URL("https://html.duckduckgo.com/html/");
|
||||
url.searchParams.set("q", query);
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
"User-Agent": "Mozilla/5.0 (compatible; VoidBot Face Eyes/0.1; +https://github.com/GameCult/VoidBot)",
|
||||
Accept: "text/html,application/xhtml+xml",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Web search failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.text();
|
||||
}
|
||||
|
||||
function parseDuckDuckGoResults(
|
||||
html: string,
|
||||
limit: number,
|
||||
): Array<{ title: string; url: string; snippet?: string }> {
|
||||
const results: Array<{ title: string; url: string; snippet?: string }> = [];
|
||||
const resultPattern = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = resultPattern.exec(html)) && results.length < limit) {
|
||||
const rawUrl = decodeHtml(match[1] ?? "");
|
||||
const title = normalizeWhitespace(stripHtml(match[2] ?? ""));
|
||||
const snippet = normalizeWhitespace(stripHtml(match[3] ?? ""));
|
||||
const resolvedUrl = normalizeDuckDuckGoResultUrl(rawUrl);
|
||||
if (title && resolvedUrl) {
|
||||
results.push({
|
||||
title,
|
||||
url: resolvedUrl,
|
||||
...(snippet ? { snippet } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function normalizeDuckDuckGoResultUrl(rawUrl: string): string | undefined {
|
||||
try {
|
||||
const parsed = rawUrl.startsWith("//") ? new URL(`https:${rawUrl}`) : new URL(rawUrl);
|
||||
const redirected = parsed.searchParams.get("uddg");
|
||||
const resolved = redirected ? decodeURIComponent(redirected) : parsed.toString();
|
||||
if (resolved.includes("duckduckgo.com/y.js?")) {
|
||||
return undefined;
|
||||
}
|
||||
return resolved;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function stripHtml(value: string): string {
|
||||
return decodeHtml(value.replace(/<[^>]*>/g, " "));
|
||||
}
|
||||
|
||||
function normalizeWhitespace(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function decodeHtml(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, "&")
|
||||
.replace(/"/g, "\"")
|
||||
.replace(/'/g, "'")
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/ /g, " ");
|
||||
}
|
||||
|
||||
async function resolveRepoIdentityForTool(
|
||||
context: VoidbotMcpContext,
|
||||
identitySelector: string,
|
||||
|
|
|
|||
Loading…
Reference in a new issue