fix(runtime): stabilize standalone container search

This commit is contained in:
xixu-me committed 2026-03-30 21:40:36 +08:00
1 parent 1542a67ba3
commit 271901221b
14 files changed
+380 -48

No files matched your search

+6 -4
View File
@@ -12,6 +12,9 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd -r xread \
&& useradd -g xread -G audio,video -m xread
WORKDIR /app
COPY package.json package-lock.json ./
@@ -28,15 +31,14 @@ RUN NODE_ENV=dry-run node ./build/stand-alone/crawl.js \
&& NODE_ENV=dry-run node ./build/stand-alone/search.js \
&& NODE_ENV=dry-run node ./build/stand-alone/serp.js
RUN groupadd -r xread \
&& useradd -g xread -G audio,video -m xread \
&& chown -R xread:xread /app
USER xread
ENV LOCAL_DB_ROOT=/tmp/xread/db
ENV OVERRIDE_CHROME_EXECUTABLE_PATH=/usr/bin/google-chrome-stable
ENV NODE_COMPILE_CACHE=node_modules
ENV PORT=8080
ENV STORAGE_ROOT=/tmp/xread/storage
ENV STANDALONE_ALLOW_INTERNAL_DNS_REWRITE=true
EXPOSE 3000 3001 8080 8081
ENTRYPOINT ["node"]
+39 -8
View File
@@ -19,6 +19,7 @@ import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
import { OutputServerEventStream } from '../lib/transform-server-event-stream';
import { ApiTokenAccount, AuthDTO } from '../dto/auth';
import { GoogleSERP } from '../services/serp/google';
import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';
import { toAsyncGenerator } from '../utils/misc';
import { LRUCache } from 'lru-cache';
@@ -26,6 +27,8 @@ import { API_CALL_STATUS } from '../shared/db/api-roll';
import { SERPResult } from '../db/searched';
import { SerperSearchQueryParams } from '../shared/3rd-party/serper-search';
import { WebSearchEntry } from '../services/serp/compat';
import { SecretExposer } from '../shared/services/secrets';
import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback';
interface FormattedPage extends RealFormattedPage {
favicon?: string;
@@ -62,8 +65,11 @@ export class SearcherHost extends RPCHost {
protected globalLogger: GlobalLogger,
protected rateLimitControl: RateLimitControl,
protected threadLocal: AsyncLocalContext,
protected secretExposer: SecretExposer,
protected crawler: CrawlerHost,
protected snapshotFormatter: SnapshotFormatter,
protected googleSerp: GoogleSERP,
protected standaloneFallback: StandaloneSearchFallbackService,
protected serperGoogle: SerperGoogleSearchService,
protected serperBing: SerperBingSearchService,
) {
@@ -712,24 +718,49 @@ export class SearcherHost extends RPCHost {
}
*iterProviders(preference?: string, _variant?: string) {
const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY;
const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY);
if (preference === 'bing') {
yield this.serperBing;
yield this.serperGoogle;
yield this.serperGoogle;
if (preferScraping) {
yield this.standaloneFallback;
}
if (includeSerper) {
yield this.serperBing;
yield this.serperGoogle;
}
yield this.googleSerp;
if (includeSerper) {
yield this.serperGoogle;
}
return;
}
if (preference === 'google') {
yield this.serperGoogle;
yield this.serperGoogle;
if (preferScraping) {
yield this.standaloneFallback;
yield this.googleSerp;
}
yield this.googleSerp;
if (includeSerper) {
yield this.serperGoogle;
yield this.serperGoogle;
}
return;
}
yield this.serperGoogle;
yield this.serperGoogle;
yield this.serperBing;
if (preferScraping) {
yield this.standaloneFallback;
yield this.googleSerp;
}
yield this.googleSerp;
if (includeSerper) {
yield this.serperGoogle;
yield this.serperGoogle;
yield this.serperBing;
}
}
async cachedSearch(variant: 'web' | 'news' | 'images', query: Record<string, any>, noCache?: boolean): Promise<WebSearchEntry[]> {
+29 -5
View File
@@ -24,6 +24,8 @@ import { SERPResult } from '../db/searched';
import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';
import { LRUCache } from 'lru-cache';
import { API_CALL_STATUS } from '../shared/db/api-roll';
import { SecretExposer } from '../shared/services/secrets';
import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback';
type RateLimitCache = {
blockedUntil?: Date;
@@ -83,7 +85,9 @@ export class SerpHost extends RPCHost {
protected globalLogger: GlobalLogger,
protected rateLimitControl: RateLimitControl,
protected threadLocal: AsyncLocalContext,
protected secretExposer: SecretExposer,
protected googleSerp: GoogleSERP,
protected standaloneFallback: StandaloneSearchFallbackService,
protected serperGoogle: SerperGoogleSearchService,
protected serperBing: SerperBingSearchService,
) {
@@ -448,25 +452,45 @@ export class SerpHost extends RPCHost {
}
*iterProviders(preference?: string, _variant?: string) {
const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY;
const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY);
if (preference === 'bing') {
yield this.serperBing;
yield this.serperGoogle;
if (preferScraping) {
yield this.standaloneFallback;
}
if (includeSerper) {
yield this.serperBing;
yield this.serperGoogle;
}
yield this.googleSerp;
return;
}
if (preference === 'google') {
if (preferScraping) {
yield this.standaloneFallback;
yield this.googleSerp;
}
yield this.googleSerp;
yield this.googleSerp;
yield this.serperGoogle;
if (includeSerper) {
yield this.serperGoogle;
}
return;
}
yield this.serperGoogle;
yield this.serperGoogle;
if (preferScraping) {
yield this.standaloneFallback;
yield this.googleSerp;
}
yield this.googleSerp;
if (includeSerper) {
yield this.serperGoogle;
yield this.serperGoogle;
}
}
async cachedSearch(variant: 'web' | 'news' | 'images', query: Record<string, any>, opts: CrawlerOptions) {
+30 -7
View File
@@ -41,21 +41,44 @@ export class CanvasService extends AsyncService {
super(...arguments);
}
protected async loadBundledFont() {
const bundledFontPath = path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf');
try {
return {
family: 'Source Han Sans SC',
data: await readFile(bundledFontPath),
};
} catch (err: any) {
if (err?.code === 'ENOENT') {
this.logger.warn(`Bundled font asset unavailable, falling back to system fonts`, {
err: { code: 'ENOENT', path: bundledFontPath },
});
return {
family: 'WenQuanYi Zen Hei',
data: undefined,
};
}
throw err;
}
}
override async init() {
await this.dependencyReady();
if (!isMainThread) {
const { createSvg2png, initialize } = require('svg2png-wasm');
const wasmBuff = await readFile(path.resolve(path.dirname(require.resolve('svg2png-wasm')), '../svg2png_wasm_bg.wasm'));
const fontBuff = await readFile(path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf'));
const font = await this.loadBundledFont();
await initialize(wasmBuff);
this.svg2png = createSvg2png({
fonts: [Uint8Array.from(fontBuff)],
fonts: font.data ? [Uint8Array.from(font.data)] : [],
defaultFontFamily: {
serifFamily: 'Source Han Sans SC',
sansSerifFamily: 'Source Han Sans SC',
cursiveFamily: 'Source Han Sans SC',
fantasyFamily: 'Source Han Sans SC',
monospaceFamily: 'Source Han Sans SC',
serifFamily: font.family,
sansSerifFamily: font.family,
cursiveFamily: font.family,
fantasyFamily: font.family,
monospaceFamily: font.family,
}
});
}
+24 -2
View File
@@ -10,6 +10,16 @@ import { Threaded } from './threaded';
const normalizeUrl = require('@esm2cjs/normalize-url').default;
const STANDALONE_ALLOW_INTERNAL_DNS_REWRITE = process.env.STANDALONE_ALLOW_INTERNAL_DNS_REWRITE === 'true';
function isStandaloneDnsRewriteAddress(address: string) {
const normalized = address.toLowerCase();
return normalized.startsWith('198.18.')
|| normalized.startsWith('198.19.')
|| normalized.startsWith('fd00::');
}
@singleton()
export class MiscService extends AsyncService {
@@ -84,8 +94,20 @@ export class MiscService extends AsyncService {
return;
});
if (resolved) {
const resolvedAddrs = resolved.map((x) => x.address);
const allowStandaloneDnsRewrite = STANDALONE_ALLOW_INTERNAL_DNS_REWRITE
&& resolvedAddrs.length
&& resolvedAddrs.every((x) => isStandaloneDnsRewriteAddress(x));
if (allowStandaloneDnsRewrite) {
this.logger.warn(`Allowing Docker DNS rewrite addresses for standalone container`, {
href: result.href,
ips: resolvedAddrs,
});
}
for (const x of resolved) {
if (isIPInNonPublicRange(x.address)) {
if (isIPInNonPublicRange(x.address) && !allowStandaloneDnsRewrite) {
this.logger.warn(`Suspicious action: Domain resolved to non-public IP: ${result.hostname} => ${x.address}`, { href: result.href, ip: x.address });
throw new SecurityCompromiseError({
message: `Suspicious action: Domain resolved to non-public IP: ${x.address}`,
@@ -104,4 +126,4 @@ export class MiscService extends AsyncService {
};
}
}
}
+17 -5
View File
@@ -24,6 +24,20 @@ const tldExtract = require('tld-extract');
const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8');
function getChromeLaunchArgs() {
const args = [
'--disable-dev-shm-usage',
'--disable-blink-features=AutomationControlled',
];
// Docker and many CI kernels do not permit Chrome's sandbox namespaces.
if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) {
args.push('--no-sandbox', '--disable-setuid-sandbox');
}
return args;
}
export interface ImgBrief {
src: string;
@@ -559,13 +573,11 @@ export class PuppeteerControl extends AsyncService {
}
}
this.browser = await puppeteer.launch({
timeout: 10_000,
timeout: 30_000,
headless: !Boolean(process.env.DEBUG_BROWSER),
pipe: true,
executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH,
args: [
'--disable-dev-shm-usage',
'--disable-blink-features=AutomationControlled'
]
args: getChromeLaunchArgs(),
}).catch((err: any) => {
this.logger.error(`Unknown firebase issue, just die fast.`, { err });
process.nextTick(() => {
+5
View File
@@ -55,6 +55,11 @@ export class GoogleSERP extends AsyncService {
return this.curlControl.sideLoad(url, opts);
}
if (!this.proxyProvider.isConfigured()) {
this.logger.info(`No proxy configured, falling back to direct Google request`, { url: url.href });
return this.curlControl.sideLoad(url, opts);
}
const proxy = await this.proxyProvider.alloc(
process.env.PREFERRED_PROXY_COUNTRY || 'auto'
);
+31 -15
View File
@@ -1,4 +1,5 @@
import _ from 'lodash';
import fs from 'fs';
import { readFile } from 'fs/promises';
import { container, singleton } from 'tsyringe';
@@ -19,6 +20,20 @@ import { GlobalLogger } from '../logger';
import { minimalStealth } from '../minimal-stealth';
import { BlackHoleDetector } from '../blackhole-detector';
function getChromeLaunchArgs() {
const args = [
'--disable-dev-shm-usage',
'--disable-blink-features=AutomationControlled',
];
// Docker and many CI kernels do not permit Chrome's sandbox namespaces.
if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) {
args.push('--no-sandbox', '--disable-setuid-sandbox');
}
return args;
}
export interface ScrappingOptions {
proxyUrl?: string;
@@ -277,12 +292,11 @@ export class SERPSpecializedPuppeteerControl extends AsyncService {
}
}
this.browser = await puppeteer.launch({
timeout: 10_000,
timeout: 30_000,
headless: !Boolean(process.env.DEBUG_BROWSER),
pipe: true,
executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH,
args: [
'--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'
]
args: getChromeLaunchArgs(),
}).catch((err: any) => {
this.logger.error(`Unknown firebase issue, just die fast.`, { err });
process.nextTick(() => {
@@ -647,9 +661,9 @@ func().then((result) => {
);
});
const timeout = options.timeoutMs || 30_000;
const timeout = options.timeoutMs || 45_000;
const goToOptions: GoToOptions = {
waitUntil: ['load', 'domcontentloaded', 'networkidle0'],
waitUntil: ['domcontentloaded', 'load'],
timeout,
};
@@ -657,12 +671,11 @@ func().then((result) => {
goToOptions.referer = options.referer;
}
const gotoPromise = page.goto(url, goToOptions)
.catch((err) => {
if (err instanceof TimeoutError) {
this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err });
return new AssertionFailureError({
throw new AssertionFailureError({
message: `Failed to goto ${url}: ${err}`,
cause: err,
});
@@ -670,17 +683,20 @@ func().then((result) => {
this.logger.warn(`Page ${sn}: Browsing of ${url} aborted`, { err });
return undefined;
}).then(async (r) => {
await delay(5000);
resultDeferred.reject(new TimeoutError(`Control function did not respond in time`));
return r;
});
try {
await Promise.race([resultDeferred.promise, gotoPromise]);
const controlTimeout = setTimeout(() => {
resultDeferred.reject(new TimeoutError(`Control function did not respond in time`));
}, timeout);
controlTimeout.unref();
return resultDeferred.promise;
try {
return await Promise.race([
resultDeferred.promise,
gotoPromise.then(() => resultDeferred.promise),
]);
} finally {
clearTimeout(controlTimeout);
page.off(this._REPORT_FUNCTION_NAME, hdl as any);
this.ditchPage(page);
resultDeferred.resolve();
+110
View File
@@ -0,0 +1,110 @@
import { singleton } from 'tsyringe';
import { AsyncService } from 'civkit/async-service';
import { GlobalLogger } from '../logger';
import { JSDomControl } from '../jsdom';
import { WebSearchEntry } from './compat';
import { ServiceBadApproachError, ServiceBadAttemptError } from '../errors';
function decodeDuckDuckGoTarget(href: string) {
try {
const url = new URL(href, 'https://html.duckduckgo.com');
const actual = url.searchParams.get('uddg');
if (actual) {
return decodeURIComponent(actual);
}
return url.toString();
} catch {
return href;
}
}
function isAdLikeDuckDuckGoTarget(target: string) {
try {
const url = new URL(target);
if (url.hostname === 'duckduckgo.com' && url.pathname === '/y.js') {
return true;
}
return url.searchParams.has('ad_domain') || url.searchParams.has('ad_provider');
} catch {
return /ad_domain=|ad_provider=/.test(target);
}
}
@singleton()
export class StandaloneSearchFallbackService extends AsyncService {
logger = this.globalLogger.child({ service: this.constructor.name });
constructor(
protected globalLogger: GlobalLogger,
protected jsDomControl: JSDomControl,
) {
super(...arguments);
}
override async init() {
await this.dependencyReady();
this.emit('ready');
}
async webSearch(query: { [k: string]: any; }) {
const url = new URL('https://html.duckduckgo.com/html/');
url.searchParams.set('q', query.q || '');
const response = await fetch(url, {
headers: {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
'accept-language': query.hl || 'en-US,en;q=0.9',
},
signal: AbortSignal.timeout(20_000),
}).catch((err) => {
throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback failed: ${err instanceof Error ? err.message : err}`);
});
if (!response.ok) {
throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback returned ${response.status}`);
}
const html = await response.text();
const { document } = this.jsDomControl.linkedom.parseHTML(html).window;
const nodes = Array.from(document.querySelectorAll('.result:not(.result--ad)'));
const results = nodes.map((node) => {
const linkElem = node.querySelector('.result__title a[href], .result__a[href]');
const title = linkElem?.textContent?.replace(/\s+/g, ' ').trim();
const href = linkElem?.getAttribute('href');
if (!title || !href) {
return undefined;
}
const link = decodeDuckDuckGoTarget(href);
if (isAdLikeDuckDuckGoTarget(link)) {
return undefined;
}
const snippet = node.querySelector('.result__snippet, .result-snippet')?.textContent?.replace(/\s+/g, ' ').trim();
const source = node.querySelector('.result__url, .result__extras__url')?.textContent?.replace(/\s+/g, ' ').trim();
return {
link,
title,
snippet: snippet || undefined,
source: source || undefined,
variant: 'web',
} as WebSearchEntry;
}).filter(Boolean) as WebSearchEntry[];
if (!results.length) {
throw new ServiceBadAttemptError('DuckDuckGo HTML fallback returned no results.');
}
return results.slice(0, query.num || 10);
}
async newsSearch() {
throw new ServiceBadApproachError('Standalone HTML fallback does not support news search.');
}
async imageSearch() {
throw new ServiceBadApproachError('Standalone HTML fallback does not support image search.');
}
}
+1 -1
View File
@@ -75,7 +75,7 @@ function applyPatch(base: any, patch: any) {
}
class LocalStore {
rootDir = path.resolve('.cache/xread/db');
rootDir = path.resolve(process.env.LOCAL_DB_ROOT || '.cache/xread/db');
constructor() {
fs.mkdirSync(this.rootDir, { recursive: true });
+4 -1
View File
@@ -31,6 +31,10 @@ export class ProxyProviderService extends AsyncService {
return /^[a-z]{2}$/i.test(input);
}
isConfigured() {
return this.proxies.length > 0;
}
async alloc(input?: string) {
if (input === 'none') {
throw new ServiceDisabledError(`Proxy allocation is disabled for '${input}'.`);
@@ -48,4 +52,3 @@ export class ProxyProviderService extends AsyncService {
yield await this.alloc(input);
}
}
+5
View File
@@ -7,6 +7,7 @@ type EnvConfig = Record<string, string> & {
readonly CLOUD_FLARE_API_KEY: string;
readonly LOCAL_PROXY_URLS: string;
readonly STORAGE_ROOT: string;
readonly LOCAL_DB_ROOT: string;
};
export const readEnv = (key: string) => process.env[key] || '';
@@ -43,6 +44,10 @@ export class SecretExposer extends AsyncService {
get STORAGE_ROOT() {
return readEnv('STORAGE_ROOT');
}
get LOCAL_DB_ROOT() {
return readEnv('LOCAL_DB_ROOT');
}
}
export default envConfig;
@@ -0,0 +1,76 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const projectRoot = path.resolve(__dirname, '..');
function readProjectFile(relativePath) {
return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8');
}
test('browser launchers include container-safe Chrome sandbox flags', () => {
const mainPuppeteer = readProjectFile('src/services/puppeteer.ts');
const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts');
for (const source of [mainPuppeteer, serpPuppeteer]) {
assert.match(source, /pipe:\s*true/);
assert.match(source, /--no-sandbox/);
assert.match(source, /--disable-setuid-sandbox/);
}
});
test('SERP puppeteer control uses an absolute control timeout instead of a fixed post-navigation race', () => {
const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts');
assert.match(serpPuppeteer, /waitUntil:\s*\['domcontentloaded', 'load'\]/);
assert.match(serpPuppeteer, /setTimeout\(\(\) => \{\s*resultDeferred\.reject\(new TimeoutError/);
assert.doesNotMatch(serpPuppeteer, /networkidle0/);
assert.doesNotMatch(serpPuppeteer, /await delay\(5000\)/);
});
test('canvas service falls back when bundled font asset is missing', () => {
const canvasService = readProjectFile('src/services/canvas.ts');
assert.match(canvasService, /ENOENT/);
assert.match(canvasService, /Bundled font asset unavailable/);
});
test('search host includes Google scraping fallback for standalone builds', () => {
const searcherHost = readProjectFile('src/api/searcher.ts');
assert.match(searcherHost, /protected googleSerp: GoogleSERP/);
assert.match(searcherHost, /protected standaloneFallback: StandaloneSearchFallbackService/);
assert.match(searcherHost, /yield this\.googleSerp/);
assert.match(searcherHost, /yield this\.standaloneFallback/);
assert.match(searcherHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/);
});
test('SERP host skips Serper providers when no API key is configured', () => {
const serpHost = readProjectFile('src/api/serp.ts');
assert.match(serpHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/);
assert.match(serpHost, /yield this\.standaloneFallback/);
});
test('standalone fallback search provider is available for local web search without private APIs', () => {
const fallbackProvider = readProjectFile('src/services/serp/standalone-fallback.ts');
assert.match(fallbackProvider, /DuckDuckGo HTML fallback/i);
assert.match(fallbackProvider, /querySelectorAll\('\.result:not\(\.result--ad\)'\)/);
assert.match(fallbackProvider, /ad_domain/);
});
test('google search provider can fall back to direct requests when no proxy is configured', () => {
const googleSerp = readProjectFile('src/services/serp/google.ts');
assert.match(googleSerp, /falling back to direct Google request/i);
});
test('standalone URL validation can allow Docker DNS rewrite ranges behind an explicit env gate', () => {
const miscService = readProjectFile('src/services/misc.ts');
assert.match(miscService, /STANDALONE_ALLOW_INTERNAL_DNS_REWRITE/);
assert.match(miscService, /198\.18/);
assert.match(miscService, /fd00::/i);
});
+3
View File
@@ -92,5 +92,8 @@ test('Dockerfile is self-contained and no longer relies on curl-impersonate', ()
assert.match(dockerfile, /FROM node:22-bookworm-slim/);
assert.match(dockerfile, /PUPPETEER_SKIP_DOWNLOAD=true/);
assert.match(dockerfile, /LOCAL_DB_ROOT=\/tmp\/xread\/db/);
assert.match(dockerfile, /STORAGE_ROOT=\/tmp\/xread\/storage/);
assert.doesNotMatch(dockerfile, /chown -R xread:xread \/app/);
assert.doesNotMatch(dockerfile, /curl-impersonate|LD_PRELOAD/);
});