From 271901221bc1d00906e5be1898412ff9da192195 Mon Sep 17 00:00:00 2001 From: Xi Xu Date: Mon, 30 Mar 2026 21:39:59 +0800 Subject: [PATCH] fix(runtime): stabilize standalone container search --- Dockerfile | 10 +- src/api/searcher.ts | 47 ++++++-- src/api/serp.ts | 34 +++++- src/services/canvas.ts | 37 +++++-- src/services/misc.ts | 26 ++++- src/services/puppeteer.ts | 22 +++- src/services/serp/google.ts | 5 + src/services/serp/puppeteer.ts | 46 +++++--- src/services/serp/standalone-fallback.ts | 110 +++++++++++++++++++ src/shared/lib/firestore.ts | 2 +- src/shared/services/proxy-provider.ts | 5 +- src/shared/services/secrets.ts | 5 + tests/container-runtime-regressions.test.cjs | 76 +++++++++++++ tests/github-automation.test.cjs | 3 + 14 files changed, 380 insertions(+), 48 deletions(-) create mode 100644 src/services/serp/standalone-fallback.ts create mode 100644 tests/container-runtime-regressions.test.cjs diff --git a/Dockerfile b/Dockerfile index 896c9f7..d56feb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,9 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd \ && rm -rf /var/lib/apt/lists/* +RUN groupadd -r xread \ + && useradd -g xread -G audio,video -m xread + WORKDIR /app COPY package.json package-lock.json ./ @@ -28,15 +31,14 @@ RUN NODE_ENV=dry-run node ./build/stand-alone/crawl.js \ && NODE_ENV=dry-run node ./build/stand-alone/search.js \ && NODE_ENV=dry-run node ./build/stand-alone/serp.js -RUN groupadd -r xread \ - && useradd -g xread -G audio,video -m xread \ - && chown -R xread:xread /app - USER xread +ENV LOCAL_DB_ROOT=/tmp/xread/db ENV OVERRIDE_CHROME_EXECUTABLE_PATH=/usr/bin/google-chrome-stable ENV NODE_COMPILE_CACHE=node_modules ENV PORT=8080 +ENV STORAGE_ROOT=/tmp/xread/storage +ENV STANDALONE_ALLOW_INTERNAL_DNS_REWRITE=true EXPOSE 3000 3001 8080 8081 ENTRYPOINT ["node"] diff --git a/src/api/searcher.ts b/src/api/searcher.ts index 2ed916a..3f51c43 100644 --- a/src/api/searcher.ts +++ b/src/api/searcher.ts @@ -19,6 +19,7 @@ import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry'; import { OutputServerEventStream } from '../lib/transform-server-event-stream'; import { ApiTokenAccount, AuthDTO } from '../dto/auth'; +import { GoogleSERP } from '../services/serp/google'; import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper'; import { toAsyncGenerator } from '../utils/misc'; import { LRUCache } from 'lru-cache'; @@ -26,6 +27,8 @@ import { API_CALL_STATUS } from '../shared/db/api-roll'; import { SERPResult } from '../db/searched'; import { SerperSearchQueryParams } from '../shared/3rd-party/serper-search'; import { WebSearchEntry } from '../services/serp/compat'; +import { SecretExposer } from '../shared/services/secrets'; +import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback'; interface FormattedPage extends RealFormattedPage { favicon?: string; @@ -62,8 +65,11 @@ export class SearcherHost extends RPCHost { protected globalLogger: GlobalLogger, protected rateLimitControl: RateLimitControl, protected threadLocal: AsyncLocalContext, + protected secretExposer: SecretExposer, protected crawler: CrawlerHost, protected snapshotFormatter: SnapshotFormatter, + protected googleSerp: GoogleSERP, + protected standaloneFallback: StandaloneSearchFallbackService, protected serperGoogle: SerperGoogleSearchService, protected serperBing: SerperBingSearchService, ) { @@ -712,24 +718,49 @@ export class SearcherHost extends RPCHost { } *iterProviders(preference?: string, _variant?: string) { + const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY; + const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY); + if (preference === 'bing') { - yield this.serperBing; - yield this.serperGoogle; - yield this.serperGoogle; + if (preferScraping) { + yield this.standaloneFallback; + } + if (includeSerper) { + yield this.serperBing; + yield this.serperGoogle; + } + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + } return; } if (preference === 'google') { - yield this.serperGoogle; - yield this.serperGoogle; + if (preferScraping) { + yield this.standaloneFallback; + yield this.googleSerp; + } + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + yield this.serperGoogle; + } return; } - yield this.serperGoogle; - yield this.serperGoogle; - yield this.serperBing; + if (preferScraping) { + yield this.standaloneFallback; + yield this.googleSerp; + } + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + yield this.serperGoogle; + yield this.serperBing; + } } async cachedSearch(variant: 'web' | 'news' | 'images', query: Record, noCache?: boolean): Promise { diff --git a/src/api/serp.ts b/src/api/serp.ts index 9446f54..2005bf6 100644 --- a/src/api/serp.ts +++ b/src/api/serp.ts @@ -24,6 +24,8 @@ import { SERPResult } from '../db/searched'; import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper'; import { LRUCache } from 'lru-cache'; import { API_CALL_STATUS } from '../shared/db/api-roll'; +import { SecretExposer } from '../shared/services/secrets'; +import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback'; type RateLimitCache = { blockedUntil?: Date; @@ -83,7 +85,9 @@ export class SerpHost extends RPCHost { protected globalLogger: GlobalLogger, protected rateLimitControl: RateLimitControl, protected threadLocal: AsyncLocalContext, + protected secretExposer: SecretExposer, protected googleSerp: GoogleSERP, + protected standaloneFallback: StandaloneSearchFallbackService, protected serperGoogle: SerperGoogleSearchService, protected serperBing: SerperBingSearchService, ) { @@ -448,25 +452,45 @@ export class SerpHost extends RPCHost { } *iterProviders(preference?: string, _variant?: string) { + const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY; + const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY); + if (preference === 'bing') { - yield this.serperBing; - yield this.serperGoogle; + if (preferScraping) { + yield this.standaloneFallback; + } + if (includeSerper) { + yield this.serperBing; + yield this.serperGoogle; + } yield this.googleSerp; return; } if (preference === 'google') { + if (preferScraping) { + yield this.standaloneFallback; + yield this.googleSerp; + } yield this.googleSerp; yield this.googleSerp; - yield this.serperGoogle; + if (includeSerper) { + yield this.serperGoogle; + } return; } - yield this.serperGoogle; - yield this.serperGoogle; + if (preferScraping) { + yield this.standaloneFallback; + yield this.googleSerp; + } yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + yield this.serperGoogle; + } } async cachedSearch(variant: 'web' | 'news' | 'images', query: Record, opts: CrawlerOptions) { diff --git a/src/services/canvas.ts b/src/services/canvas.ts index 15db6e9..1853a0d 100644 --- a/src/services/canvas.ts +++ b/src/services/canvas.ts @@ -41,21 +41,44 @@ export class CanvasService extends AsyncService { super(...arguments); } + protected async loadBundledFont() { + const bundledFontPath = path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf'); + + try { + return { + family: 'Source Han Sans SC', + data: await readFile(bundledFontPath), + }; + } catch (err: any) { + if (err?.code === 'ENOENT') { + this.logger.warn(`Bundled font asset unavailable, falling back to system fonts`, { + err: { code: 'ENOENT', path: bundledFontPath }, + }); + return { + family: 'WenQuanYi Zen Hei', + data: undefined, + }; + } + + throw err; + } + } + override async init() { await this.dependencyReady(); if (!isMainThread) { const { createSvg2png, initialize } = require('svg2png-wasm'); const wasmBuff = await readFile(path.resolve(path.dirname(require.resolve('svg2png-wasm')), '../svg2png_wasm_bg.wasm')); - const fontBuff = await readFile(path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf')); + const font = await this.loadBundledFont(); await initialize(wasmBuff); this.svg2png = createSvg2png({ - fonts: [Uint8Array.from(fontBuff)], + fonts: font.data ? [Uint8Array.from(font.data)] : [], defaultFontFamily: { - serifFamily: 'Source Han Sans SC', - sansSerifFamily: 'Source Han Sans SC', - cursiveFamily: 'Source Han Sans SC', - fantasyFamily: 'Source Han Sans SC', - monospaceFamily: 'Source Han Sans SC', + serifFamily: font.family, + sansSerifFamily: font.family, + cursiveFamily: font.family, + fantasyFamily: font.family, + monospaceFamily: font.family, } }); } diff --git a/src/services/misc.ts b/src/services/misc.ts index 82ea5b5..6fa2012 100644 --- a/src/services/misc.ts +++ b/src/services/misc.ts @@ -10,6 +10,16 @@ import { Threaded } from './threaded'; const normalizeUrl = require('@esm2cjs/normalize-url').default; +const STANDALONE_ALLOW_INTERNAL_DNS_REWRITE = process.env.STANDALONE_ALLOW_INTERNAL_DNS_REWRITE === 'true'; + +function isStandaloneDnsRewriteAddress(address: string) { + const normalized = address.toLowerCase(); + + return normalized.startsWith('198.18.') + || normalized.startsWith('198.19.') + || normalized.startsWith('fd00::'); +} + @singleton() export class MiscService extends AsyncService { @@ -84,8 +94,20 @@ export class MiscService extends AsyncService { return; }); if (resolved) { + const resolvedAddrs = resolved.map((x) => x.address); + const allowStandaloneDnsRewrite = STANDALONE_ALLOW_INTERNAL_DNS_REWRITE + && resolvedAddrs.length + && resolvedAddrs.every((x) => isStandaloneDnsRewriteAddress(x)); + + if (allowStandaloneDnsRewrite) { + this.logger.warn(`Allowing Docker DNS rewrite addresses for standalone container`, { + href: result.href, + ips: resolvedAddrs, + }); + } + for (const x of resolved) { - if (isIPInNonPublicRange(x.address)) { + if (isIPInNonPublicRange(x.address) && !allowStandaloneDnsRewrite) { this.logger.warn(`Suspicious action: Domain resolved to non-public IP: ${result.hostname} => ${x.address}`, { href: result.href, ip: x.address }); throw new SecurityCompromiseError({ message: `Suspicious action: Domain resolved to non-public IP: ${x.address}`, @@ -104,4 +126,4 @@ export class MiscService extends AsyncService { }; } -} \ No newline at end of file +} diff --git a/src/services/puppeteer.ts b/src/services/puppeteer.ts index 833d3bd..fd726f2 100644 --- a/src/services/puppeteer.ts +++ b/src/services/puppeteer.ts @@ -24,6 +24,20 @@ const tldExtract = require('tld-extract'); const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8'); +function getChromeLaunchArgs() { + const args = [ + '--disable-dev-shm-usage', + '--disable-blink-features=AutomationControlled', + ]; + + // Docker and many CI kernels do not permit Chrome's sandbox namespaces. + if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) { + args.push('--no-sandbox', '--disable-setuid-sandbox'); + } + + return args; +} + export interface ImgBrief { src: string; @@ -559,13 +573,11 @@ export class PuppeteerControl extends AsyncService { } } this.browser = await puppeteer.launch({ - timeout: 10_000, + timeout: 30_000, headless: !Boolean(process.env.DEBUG_BROWSER), + pipe: true, executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH, - args: [ - '--disable-dev-shm-usage', - '--disable-blink-features=AutomationControlled' - ] + args: getChromeLaunchArgs(), }).catch((err: any) => { this.logger.error(`Unknown firebase issue, just die fast.`, { err }); process.nextTick(() => { diff --git a/src/services/serp/google.ts b/src/services/serp/google.ts index 8f8ae3f..25e6af5 100644 --- a/src/services/serp/google.ts +++ b/src/services/serp/google.ts @@ -55,6 +55,11 @@ export class GoogleSERP extends AsyncService { return this.curlControl.sideLoad(url, opts); } + if (!this.proxyProvider.isConfigured()) { + this.logger.info(`No proxy configured, falling back to direct Google request`, { url: url.href }); + return this.curlControl.sideLoad(url, opts); + } + const proxy = await this.proxyProvider.alloc( process.env.PREFERRED_PROXY_COUNTRY || 'auto' ); diff --git a/src/services/serp/puppeteer.ts b/src/services/serp/puppeteer.ts index f7ba523..2616339 100644 --- a/src/services/serp/puppeteer.ts +++ b/src/services/serp/puppeteer.ts @@ -1,4 +1,5 @@ import _ from 'lodash'; +import fs from 'fs'; import { readFile } from 'fs/promises'; import { container, singleton } from 'tsyringe'; @@ -19,6 +20,20 @@ import { GlobalLogger } from '../logger'; import { minimalStealth } from '../minimal-stealth'; import { BlackHoleDetector } from '../blackhole-detector'; +function getChromeLaunchArgs() { + const args = [ + '--disable-dev-shm-usage', + '--disable-blink-features=AutomationControlled', + ]; + + // Docker and many CI kernels do not permit Chrome's sandbox namespaces. + if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) { + args.push('--no-sandbox', '--disable-setuid-sandbox'); + } + + return args; +} + export interface ScrappingOptions { proxyUrl?: string; @@ -277,12 +292,11 @@ export class SERPSpecializedPuppeteerControl extends AsyncService { } } this.browser = await puppeteer.launch({ - timeout: 10_000, + timeout: 30_000, headless: !Boolean(process.env.DEBUG_BROWSER), + pipe: true, executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH, - args: [ - '--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled' - ] + args: getChromeLaunchArgs(), }).catch((err: any) => { this.logger.error(`Unknown firebase issue, just die fast.`, { err }); process.nextTick(() => { @@ -647,9 +661,9 @@ func().then((result) => { ); }); - const timeout = options.timeoutMs || 30_000; + const timeout = options.timeoutMs || 45_000; const goToOptions: GoToOptions = { - waitUntil: ['load', 'domcontentloaded', 'networkidle0'], + waitUntil: ['domcontentloaded', 'load'], timeout, }; @@ -657,12 +671,11 @@ func().then((result) => { goToOptions.referer = options.referer; } - const gotoPromise = page.goto(url, goToOptions) .catch((err) => { if (err instanceof TimeoutError) { this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err }); - return new AssertionFailureError({ + throw new AssertionFailureError({ message: `Failed to goto ${url}: ${err}`, cause: err, }); @@ -670,17 +683,20 @@ func().then((result) => { this.logger.warn(`Page ${sn}: Browsing of ${url} aborted`, { err }); return undefined; - }).then(async (r) => { - await delay(5000); - resultDeferred.reject(new TimeoutError(`Control function did not respond in time`)); - return r; }); - try { - await Promise.race([resultDeferred.promise, gotoPromise]); + const controlTimeout = setTimeout(() => { + resultDeferred.reject(new TimeoutError(`Control function did not respond in time`)); + }, timeout); + controlTimeout.unref(); - return resultDeferred.promise; + try { + return await Promise.race([ + resultDeferred.promise, + gotoPromise.then(() => resultDeferred.promise), + ]); } finally { + clearTimeout(controlTimeout); page.off(this._REPORT_FUNCTION_NAME, hdl as any); this.ditchPage(page); resultDeferred.resolve(); diff --git a/src/services/serp/standalone-fallback.ts b/src/services/serp/standalone-fallback.ts new file mode 100644 index 0000000..9994fc7 --- /dev/null +++ b/src/services/serp/standalone-fallback.ts @@ -0,0 +1,110 @@ +import { singleton } from 'tsyringe'; +import { AsyncService } from 'civkit/async-service'; +import { GlobalLogger } from '../logger'; +import { JSDomControl } from '../jsdom'; +import { WebSearchEntry } from './compat'; +import { ServiceBadApproachError, ServiceBadAttemptError } from '../errors'; + +function decodeDuckDuckGoTarget(href: string) { + try { + const url = new URL(href, 'https://html.duckduckgo.com'); + const actual = url.searchParams.get('uddg'); + if (actual) { + return decodeURIComponent(actual); + } + return url.toString(); + } catch { + return href; + } +} + +function isAdLikeDuckDuckGoTarget(target: string) { + try { + const url = new URL(target); + if (url.hostname === 'duckduckgo.com' && url.pathname === '/y.js') { + return true; + } + return url.searchParams.has('ad_domain') || url.searchParams.has('ad_provider'); + } catch { + return /ad_domain=|ad_provider=/.test(target); + } +} + +@singleton() +export class StandaloneSearchFallbackService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); + + constructor( + protected globalLogger: GlobalLogger, + protected jsDomControl: JSDomControl, + ) { + super(...arguments); + } + + override async init() { + await this.dependencyReady(); + this.emit('ready'); + } + + async webSearch(query: { [k: string]: any; }) { + const url = new URL('https://html.duckduckgo.com/html/'); + url.searchParams.set('q', query.q || ''); + + const response = await fetch(url, { + headers: { + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36', + 'accept-language': query.hl || 'en-US,en;q=0.9', + }, + signal: AbortSignal.timeout(20_000), + }).catch((err) => { + throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback failed: ${err instanceof Error ? err.message : err}`); + }); + + if (!response.ok) { + throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback returned ${response.status}`); + } + + const html = await response.text(); + const { document } = this.jsDomControl.linkedom.parseHTML(html).window; + const nodes = Array.from(document.querySelectorAll('.result:not(.result--ad)')); + const results = nodes.map((node) => { + const linkElem = node.querySelector('.result__title a[href], .result__a[href]'); + const title = linkElem?.textContent?.replace(/\s+/g, ' ').trim(); + const href = linkElem?.getAttribute('href'); + + if (!title || !href) { + return undefined; + } + + const link = decodeDuckDuckGoTarget(href); + if (isAdLikeDuckDuckGoTarget(link)) { + return undefined; + } + + const snippet = node.querySelector('.result__snippet, .result-snippet')?.textContent?.replace(/\s+/g, ' ').trim(); + const source = node.querySelector('.result__url, .result__extras__url')?.textContent?.replace(/\s+/g, ' ').trim(); + + return { + link, + title, + snippet: snippet || undefined, + source: source || undefined, + variant: 'web', + } as WebSearchEntry; + }).filter(Boolean) as WebSearchEntry[]; + + if (!results.length) { + throw new ServiceBadAttemptError('DuckDuckGo HTML fallback returned no results.'); + } + + return results.slice(0, query.num || 10); + } + + async newsSearch() { + throw new ServiceBadApproachError('Standalone HTML fallback does not support news search.'); + } + + async imageSearch() { + throw new ServiceBadApproachError('Standalone HTML fallback does not support image search.'); + } +} diff --git a/src/shared/lib/firestore.ts b/src/shared/lib/firestore.ts index dc7d8e9..a510cd7 100644 --- a/src/shared/lib/firestore.ts +++ b/src/shared/lib/firestore.ts @@ -75,7 +75,7 @@ function applyPatch(base: any, patch: any) { } class LocalStore { - rootDir = path.resolve('.cache/xread/db'); + rootDir = path.resolve(process.env.LOCAL_DB_ROOT || '.cache/xread/db'); constructor() { fs.mkdirSync(this.rootDir, { recursive: true }); diff --git a/src/shared/services/proxy-provider.ts b/src/shared/services/proxy-provider.ts index 9698d96..3871876 100644 --- a/src/shared/services/proxy-provider.ts +++ b/src/shared/services/proxy-provider.ts @@ -31,6 +31,10 @@ export class ProxyProviderService extends AsyncService { return /^[a-z]{2}$/i.test(input); } + isConfigured() { + return this.proxies.length > 0; + } + async alloc(input?: string) { if (input === 'none') { throw new ServiceDisabledError(`Proxy allocation is disabled for '${input}'.`); @@ -48,4 +52,3 @@ export class ProxyProviderService extends AsyncService { yield await this.alloc(input); } } - diff --git a/src/shared/services/secrets.ts b/src/shared/services/secrets.ts index 8a51d19..54cdd42 100644 --- a/src/shared/services/secrets.ts +++ b/src/shared/services/secrets.ts @@ -7,6 +7,7 @@ type EnvConfig = Record & { readonly CLOUD_FLARE_API_KEY: string; readonly LOCAL_PROXY_URLS: string; readonly STORAGE_ROOT: string; + readonly LOCAL_DB_ROOT: string; }; export const readEnv = (key: string) => process.env[key] || ''; @@ -43,6 +44,10 @@ export class SecretExposer extends AsyncService { get STORAGE_ROOT() { return readEnv('STORAGE_ROOT'); } + + get LOCAL_DB_ROOT() { + return readEnv('LOCAL_DB_ROOT'); + } } export default envConfig; diff --git a/tests/container-runtime-regressions.test.cjs b/tests/container-runtime-regressions.test.cjs new file mode 100644 index 0000000..8af92b3 --- /dev/null +++ b/tests/container-runtime-regressions.test.cjs @@ -0,0 +1,76 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const projectRoot = path.resolve(__dirname, '..'); + +function readProjectFile(relativePath) { + return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8'); +} + +test('browser launchers include container-safe Chrome sandbox flags', () => { + const mainPuppeteer = readProjectFile('src/services/puppeteer.ts'); + const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts'); + + for (const source of [mainPuppeteer, serpPuppeteer]) { + assert.match(source, /pipe:\s*true/); + assert.match(source, /--no-sandbox/); + assert.match(source, /--disable-setuid-sandbox/); + } +}); + +test('SERP puppeteer control uses an absolute control timeout instead of a fixed post-navigation race', () => { + const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts'); + + assert.match(serpPuppeteer, /waitUntil:\s*\['domcontentloaded', 'load'\]/); + assert.match(serpPuppeteer, /setTimeout\(\(\) => \{\s*resultDeferred\.reject\(new TimeoutError/); + assert.doesNotMatch(serpPuppeteer, /networkidle0/); + assert.doesNotMatch(serpPuppeteer, /await delay\(5000\)/); +}); + +test('canvas service falls back when bundled font asset is missing', () => { + const canvasService = readProjectFile('src/services/canvas.ts'); + + assert.match(canvasService, /ENOENT/); + assert.match(canvasService, /Bundled font asset unavailable/); +}); + +test('search host includes Google scraping fallback for standalone builds', () => { + const searcherHost = readProjectFile('src/api/searcher.ts'); + + assert.match(searcherHost, /protected googleSerp: GoogleSERP/); + assert.match(searcherHost, /protected standaloneFallback: StandaloneSearchFallbackService/); + assert.match(searcherHost, /yield this\.googleSerp/); + assert.match(searcherHost, /yield this\.standaloneFallback/); + assert.match(searcherHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/); +}); + +test('SERP host skips Serper providers when no API key is configured', () => { + const serpHost = readProjectFile('src/api/serp.ts'); + + assert.match(serpHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/); + assert.match(serpHost, /yield this\.standaloneFallback/); +}); + +test('standalone fallback search provider is available for local web search without private APIs', () => { + const fallbackProvider = readProjectFile('src/services/serp/standalone-fallback.ts'); + + assert.match(fallbackProvider, /DuckDuckGo HTML fallback/i); + assert.match(fallbackProvider, /querySelectorAll\('\.result:not\(\.result--ad\)'\)/); + assert.match(fallbackProvider, /ad_domain/); +}); + +test('google search provider can fall back to direct requests when no proxy is configured', () => { + const googleSerp = readProjectFile('src/services/serp/google.ts'); + + assert.match(googleSerp, /falling back to direct Google request/i); +}); + +test('standalone URL validation can allow Docker DNS rewrite ranges behind an explicit env gate', () => { + const miscService = readProjectFile('src/services/misc.ts'); + + assert.match(miscService, /STANDALONE_ALLOW_INTERNAL_DNS_REWRITE/); + assert.match(miscService, /198\.18/); + assert.match(miscService, /fd00::/i); +}); diff --git a/tests/github-automation.test.cjs b/tests/github-automation.test.cjs index db7ba31..470ef79 100644 --- a/tests/github-automation.test.cjs +++ b/tests/github-automation.test.cjs @@ -92,5 +92,8 @@ test('Dockerfile is self-contained and no longer relies on curl-impersonate', () assert.match(dockerfile, /FROM node:22-bookworm-slim/); assert.match(dockerfile, /PUPPETEER_SKIP_DOWNLOAD=true/); + assert.match(dockerfile, /LOCAL_DB_ROOT=\/tmp\/xread\/db/); + assert.match(dockerfile, /STORAGE_ROOT=\/tmp\/xread\/storage/); + assert.doesNotMatch(dockerfile, /chown -R xread:xread \/app/); assert.doesNotMatch(dockerfile, /curl-impersonate|LD_PRELOAD/); });