fix(runtime): stabilize standalone container search
This commit is contained in:
1 parent
1542a67ba3
commit
271901221b
14 files changed
+380
-48
No files matched your search
+6
-4
@@ -12,6 +12,9 @@ RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -r xread \
|
||||
&& useradd -g xread -G audio,video -m xread
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY package.json package-lock.json ./
|
||||
@@ -28,15 +31,14 @@ RUN NODE_ENV=dry-run node ./build/stand-alone/crawl.js \
|
||||
&& NODE_ENV=dry-run node ./build/stand-alone/search.js \
|
||||
&& NODE_ENV=dry-run node ./build/stand-alone/serp.js
|
||||
|
||||
RUN groupadd -r xread \
|
||||
&& useradd -g xread -G audio,video -m xread \
|
||||
&& chown -R xread:xread /app
|
||||
|
||||
USER xread
|
||||
|
||||
ENV LOCAL_DB_ROOT=/tmp/xread/db
|
||||
ENV OVERRIDE_CHROME_EXECUTABLE_PATH=/usr/bin/google-chrome-stable
|
||||
ENV NODE_COMPILE_CACHE=node_modules
|
||||
ENV PORT=8080
|
||||
ENV STORAGE_ROOT=/tmp/xread/storage
|
||||
ENV STANDALONE_ALLOW_INTERNAL_DNS_REWRITE=true
|
||||
|
||||
EXPOSE 3000 3001 8080 8081
|
||||
ENTRYPOINT ["node"]
|
||||
|
||||
+39
-8
@@ -19,6 +19,7 @@ import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry';
|
||||
import { OutputServerEventStream } from '../lib/transform-server-event-stream';
|
||||
import { ApiTokenAccount, AuthDTO } from '../dto/auth';
|
||||
|
||||
import { GoogleSERP } from '../services/serp/google';
|
||||
import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';
|
||||
import { toAsyncGenerator } from '../utils/misc';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
@@ -26,6 +27,8 @@ import { API_CALL_STATUS } from '../shared/db/api-roll';
|
||||
import { SERPResult } from '../db/searched';
|
||||
import { SerperSearchQueryParams } from '../shared/3rd-party/serper-search';
|
||||
import { WebSearchEntry } from '../services/serp/compat';
|
||||
import { SecretExposer } from '../shared/services/secrets';
|
||||
import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback';
|
||||
|
||||
interface FormattedPage extends RealFormattedPage {
|
||||
favicon?: string;
|
||||
@@ -62,8 +65,11 @@ export class SearcherHost extends RPCHost {
|
||||
protected globalLogger: GlobalLogger,
|
||||
protected rateLimitControl: RateLimitControl,
|
||||
protected threadLocal: AsyncLocalContext,
|
||||
protected secretExposer: SecretExposer,
|
||||
protected crawler: CrawlerHost,
|
||||
protected snapshotFormatter: SnapshotFormatter,
|
||||
protected googleSerp: GoogleSERP,
|
||||
protected standaloneFallback: StandaloneSearchFallbackService,
|
||||
protected serperGoogle: SerperGoogleSearchService,
|
||||
protected serperBing: SerperBingSearchService,
|
||||
) {
|
||||
@@ -712,24 +718,49 @@ export class SearcherHost extends RPCHost {
|
||||
}
|
||||
|
||||
*iterProviders(preference?: string, _variant?: string) {
|
||||
const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY;
|
||||
const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY);
|
||||
|
||||
if (preference === 'bing') {
|
||||
yield this.serperBing;
|
||||
yield this.serperGoogle;
|
||||
yield this.serperGoogle;
|
||||
if (preferScraping) {
|
||||
yield this.standaloneFallback;
|
||||
}
|
||||
if (includeSerper) {
|
||||
yield this.serperBing;
|
||||
yield this.serperGoogle;
|
||||
}
|
||||
yield this.googleSerp;
|
||||
if (includeSerper) {
|
||||
yield this.serperGoogle;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (preference === 'google') {
|
||||
yield this.serperGoogle;
|
||||
yield this.serperGoogle;
|
||||
if (preferScraping) {
|
||||
yield this.standaloneFallback;
|
||||
yield this.googleSerp;
|
||||
}
|
||||
yield this.googleSerp;
|
||||
if (includeSerper) {
|
||||
yield this.serperGoogle;
|
||||
yield this.serperGoogle;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
yield this.serperGoogle;
|
||||
yield this.serperGoogle;
|
||||
yield this.serperBing;
|
||||
if (preferScraping) {
|
||||
yield this.standaloneFallback;
|
||||
yield this.googleSerp;
|
||||
}
|
||||
yield this.googleSerp;
|
||||
if (includeSerper) {
|
||||
yield this.serperGoogle;
|
||||
yield this.serperGoogle;
|
||||
yield this.serperBing;
|
||||
}
|
||||
}
|
||||
|
||||
async cachedSearch(variant: 'web' | 'news' | 'images', query: Record<string, any>, noCache?: boolean): Promise<WebSearchEntry[]> {
|
||||
|
||||
+29
-5
@@ -24,6 +24,8 @@ import { SERPResult } from '../db/searched';
|
||||
import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { API_CALL_STATUS } from '../shared/db/api-roll';
|
||||
import { SecretExposer } from '../shared/services/secrets';
|
||||
import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback';
|
||||
|
||||
type RateLimitCache = {
|
||||
blockedUntil?: Date;
|
||||
@@ -83,7 +85,9 @@ export class SerpHost extends RPCHost {
|
||||
protected globalLogger: GlobalLogger,
|
||||
protected rateLimitControl: RateLimitControl,
|
||||
protected threadLocal: AsyncLocalContext,
|
||||
protected secretExposer: SecretExposer,
|
||||
protected googleSerp: GoogleSERP,
|
||||
protected standaloneFallback: StandaloneSearchFallbackService,
|
||||
protected serperGoogle: SerperGoogleSearchService,
|
||||
protected serperBing: SerperBingSearchService,
|
||||
) {
|
||||
@@ -448,25 +452,45 @@ export class SerpHost extends RPCHost {
|
||||
}
|
||||
|
||||
*iterProviders(preference?: string, _variant?: string) {
|
||||
const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY;
|
||||
const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY);
|
||||
|
||||
if (preference === 'bing') {
|
||||
yield this.serperBing;
|
||||
yield this.serperGoogle;
|
||||
if (preferScraping) {
|
||||
yield this.standaloneFallback;
|
||||
}
|
||||
if (includeSerper) {
|
||||
yield this.serperBing;
|
||||
yield this.serperGoogle;
|
||||
}
|
||||
yield this.googleSerp;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
if (preference === 'google') {
|
||||
if (preferScraping) {
|
||||
yield this.standaloneFallback;
|
||||
yield this.googleSerp;
|
||||
}
|
||||
yield this.googleSerp;
|
||||
yield this.googleSerp;
|
||||
yield this.serperGoogle;
|
||||
if (includeSerper) {
|
||||
yield this.serperGoogle;
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
yield this.serperGoogle;
|
||||
yield this.serperGoogle;
|
||||
if (preferScraping) {
|
||||
yield this.standaloneFallback;
|
||||
yield this.googleSerp;
|
||||
}
|
||||
yield this.googleSerp;
|
||||
if (includeSerper) {
|
||||
yield this.serperGoogle;
|
||||
yield this.serperGoogle;
|
||||
}
|
||||
}
|
||||
|
||||
async cachedSearch(variant: 'web' | 'news' | 'images', query: Record<string, any>, opts: CrawlerOptions) {
|
||||
|
||||
+30
-7
@@ -41,21 +41,44 @@ export class CanvasService extends AsyncService {
|
||||
super(...arguments);
|
||||
}
|
||||
|
||||
protected async loadBundledFont() {
|
||||
const bundledFontPath = path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf');
|
||||
|
||||
try {
|
||||
return {
|
||||
family: 'Source Han Sans SC',
|
||||
data: await readFile(bundledFontPath),
|
||||
};
|
||||
} catch (err: any) {
|
||||
if (err?.code === 'ENOENT') {
|
||||
this.logger.warn(`Bundled font asset unavailable, falling back to system fonts`, {
|
||||
err: { code: 'ENOENT', path: bundledFontPath },
|
||||
});
|
||||
return {
|
||||
family: 'WenQuanYi Zen Hei',
|
||||
data: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
override async init() {
|
||||
await this.dependencyReady();
|
||||
if (!isMainThread) {
|
||||
const { createSvg2png, initialize } = require('svg2png-wasm');
|
||||
const wasmBuff = await readFile(path.resolve(path.dirname(require.resolve('svg2png-wasm')), '../svg2png_wasm_bg.wasm'));
|
||||
const fontBuff = await readFile(path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf'));
|
||||
const font = await this.loadBundledFont();
|
||||
await initialize(wasmBuff);
|
||||
this.svg2png = createSvg2png({
|
||||
fonts: [Uint8Array.from(fontBuff)],
|
||||
fonts: font.data ? [Uint8Array.from(font.data)] : [],
|
||||
defaultFontFamily: {
|
||||
serifFamily: 'Source Han Sans SC',
|
||||
sansSerifFamily: 'Source Han Sans SC',
|
||||
cursiveFamily: 'Source Han Sans SC',
|
||||
fantasyFamily: 'Source Han Sans SC',
|
||||
monospaceFamily: 'Source Han Sans SC',
|
||||
serifFamily: font.family,
|
||||
sansSerifFamily: font.family,
|
||||
cursiveFamily: font.family,
|
||||
fantasyFamily: font.family,
|
||||
monospaceFamily: font.family,
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
+24
-2
@@ -10,6 +10,16 @@ import { Threaded } from './threaded';
|
||||
|
||||
const normalizeUrl = require('@esm2cjs/normalize-url').default;
|
||||
|
||||
const STANDALONE_ALLOW_INTERNAL_DNS_REWRITE = process.env.STANDALONE_ALLOW_INTERNAL_DNS_REWRITE === 'true';
|
||||
|
||||
function isStandaloneDnsRewriteAddress(address: string) {
|
||||
const normalized = address.toLowerCase();
|
||||
|
||||
return normalized.startsWith('198.18.')
|
||||
|| normalized.startsWith('198.19.')
|
||||
|| normalized.startsWith('fd00::');
|
||||
}
|
||||
|
||||
@singleton()
|
||||
export class MiscService extends AsyncService {
|
||||
|
||||
@@ -84,8 +94,20 @@ export class MiscService extends AsyncService {
|
||||
return;
|
||||
});
|
||||
if (resolved) {
|
||||
const resolvedAddrs = resolved.map((x) => x.address);
|
||||
const allowStandaloneDnsRewrite = STANDALONE_ALLOW_INTERNAL_DNS_REWRITE
|
||||
&& resolvedAddrs.length
|
||||
&& resolvedAddrs.every((x) => isStandaloneDnsRewriteAddress(x));
|
||||
|
||||
if (allowStandaloneDnsRewrite) {
|
||||
this.logger.warn(`Allowing Docker DNS rewrite addresses for standalone container`, {
|
||||
href: result.href,
|
||||
ips: resolvedAddrs,
|
||||
});
|
||||
}
|
||||
|
||||
for (const x of resolved) {
|
||||
if (isIPInNonPublicRange(x.address)) {
|
||||
if (isIPInNonPublicRange(x.address) && !allowStandaloneDnsRewrite) {
|
||||
this.logger.warn(`Suspicious action: Domain resolved to non-public IP: ${result.hostname} => ${x.address}`, { href: result.href, ip: x.address });
|
||||
throw new SecurityCompromiseError({
|
||||
message: `Suspicious action: Domain resolved to non-public IP: ${x.address}`,
|
||||
@@ -104,4 +126,4 @@ export class MiscService extends AsyncService {
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,20 @@ const tldExtract = require('tld-extract');
|
||||
|
||||
const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8');
|
||||
|
||||
function getChromeLaunchArgs() {
|
||||
const args = [
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
];
|
||||
|
||||
// Docker and many CI kernels do not permit Chrome's sandbox namespaces.
|
||||
if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) {
|
||||
args.push('--no-sandbox', '--disable-setuid-sandbox');
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
|
||||
export interface ImgBrief {
|
||||
src: string;
|
||||
@@ -559,13 +573,11 @@ export class PuppeteerControl extends AsyncService {
|
||||
}
|
||||
}
|
||||
this.browser = await puppeteer.launch({
|
||||
timeout: 10_000,
|
||||
timeout: 30_000,
|
||||
headless: !Boolean(process.env.DEBUG_BROWSER),
|
||||
pipe: true,
|
||||
executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH,
|
||||
args: [
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-blink-features=AutomationControlled'
|
||||
]
|
||||
args: getChromeLaunchArgs(),
|
||||
}).catch((err: any) => {
|
||||
this.logger.error(`Unknown firebase issue, just die fast.`, { err });
|
||||
process.nextTick(() => {
|
||||
|
||||
@@ -55,6 +55,11 @@ export class GoogleSERP extends AsyncService {
|
||||
return this.curlControl.sideLoad(url, opts);
|
||||
}
|
||||
|
||||
if (!this.proxyProvider.isConfigured()) {
|
||||
this.logger.info(`No proxy configured, falling back to direct Google request`, { url: url.href });
|
||||
return this.curlControl.sideLoad(url, opts);
|
||||
}
|
||||
|
||||
const proxy = await this.proxyProvider.alloc(
|
||||
process.env.PREFERRED_PROXY_COUNTRY || 'auto'
|
||||
);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import _ from 'lodash';
|
||||
import fs from 'fs';
|
||||
import { readFile } from 'fs/promises';
|
||||
import { container, singleton } from 'tsyringe';
|
||||
|
||||
@@ -19,6 +20,20 @@ import { GlobalLogger } from '../logger';
|
||||
import { minimalStealth } from '../minimal-stealth';
|
||||
import { BlackHoleDetector } from '../blackhole-detector';
|
||||
|
||||
function getChromeLaunchArgs() {
|
||||
const args = [
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
];
|
||||
|
||||
// Docker and many CI kernels do not permit Chrome's sandbox namespaces.
|
||||
if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) {
|
||||
args.push('--no-sandbox', '--disable-setuid-sandbox');
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
|
||||
export interface ScrappingOptions {
|
||||
proxyUrl?: string;
|
||||
@@ -277,12 +292,11 @@ export class SERPSpecializedPuppeteerControl extends AsyncService {
|
||||
}
|
||||
}
|
||||
this.browser = await puppeteer.launch({
|
||||
timeout: 10_000,
|
||||
timeout: 30_000,
|
||||
headless: !Boolean(process.env.DEBUG_BROWSER),
|
||||
pipe: true,
|
||||
executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH,
|
||||
args: [
|
||||
'--disable-dev-shm-usage', '--disable-blink-features=AutomationControlled'
|
||||
]
|
||||
args: getChromeLaunchArgs(),
|
||||
}).catch((err: any) => {
|
||||
this.logger.error(`Unknown firebase issue, just die fast.`, { err });
|
||||
process.nextTick(() => {
|
||||
@@ -647,9 +661,9 @@ func().then((result) => {
|
||||
);
|
||||
});
|
||||
|
||||
const timeout = options.timeoutMs || 30_000;
|
||||
const timeout = options.timeoutMs || 45_000;
|
||||
const goToOptions: GoToOptions = {
|
||||
waitUntil: ['load', 'domcontentloaded', 'networkidle0'],
|
||||
waitUntil: ['domcontentloaded', 'load'],
|
||||
timeout,
|
||||
};
|
||||
|
||||
@@ -657,12 +671,11 @@ func().then((result) => {
|
||||
goToOptions.referer = options.referer;
|
||||
}
|
||||
|
||||
|
||||
const gotoPromise = page.goto(url, goToOptions)
|
||||
.catch((err) => {
|
||||
if (err instanceof TimeoutError) {
|
||||
this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err });
|
||||
return new AssertionFailureError({
|
||||
throw new AssertionFailureError({
|
||||
message: `Failed to goto ${url}: ${err}`,
|
||||
cause: err,
|
||||
});
|
||||
@@ -670,17 +683,20 @@ func().then((result) => {
|
||||
|
||||
this.logger.warn(`Page ${sn}: Browsing of ${url} aborted`, { err });
|
||||
return undefined;
|
||||
}).then(async (r) => {
|
||||
await delay(5000);
|
||||
resultDeferred.reject(new TimeoutError(`Control function did not respond in time`));
|
||||
return r;
|
||||
});
|
||||
|
||||
try {
|
||||
await Promise.race([resultDeferred.promise, gotoPromise]);
|
||||
const controlTimeout = setTimeout(() => {
|
||||
resultDeferred.reject(new TimeoutError(`Control function did not respond in time`));
|
||||
}, timeout);
|
||||
controlTimeout.unref();
|
||||
|
||||
return resultDeferred.promise;
|
||||
try {
|
||||
return await Promise.race([
|
||||
resultDeferred.promise,
|
||||
gotoPromise.then(() => resultDeferred.promise),
|
||||
]);
|
||||
} finally {
|
||||
clearTimeout(controlTimeout);
|
||||
page.off(this._REPORT_FUNCTION_NAME, hdl as any);
|
||||
this.ditchPage(page);
|
||||
resultDeferred.resolve();
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { singleton } from 'tsyringe';
|
||||
import { AsyncService } from 'civkit/async-service';
|
||||
import { GlobalLogger } from '../logger';
|
||||
import { JSDomControl } from '../jsdom';
|
||||
import { WebSearchEntry } from './compat';
|
||||
import { ServiceBadApproachError, ServiceBadAttemptError } from '../errors';
|
||||
|
||||
function decodeDuckDuckGoTarget(href: string) {
|
||||
try {
|
||||
const url = new URL(href, 'https://html.duckduckgo.com');
|
||||
const actual = url.searchParams.get('uddg');
|
||||
if (actual) {
|
||||
return decodeURIComponent(actual);
|
||||
}
|
||||
return url.toString();
|
||||
} catch {
|
||||
return href;
|
||||
}
|
||||
}
|
||||
|
||||
function isAdLikeDuckDuckGoTarget(target: string) {
|
||||
try {
|
||||
const url = new URL(target);
|
||||
if (url.hostname === 'duckduckgo.com' && url.pathname === '/y.js') {
|
||||
return true;
|
||||
}
|
||||
return url.searchParams.has('ad_domain') || url.searchParams.has('ad_provider');
|
||||
} catch {
|
||||
return /ad_domain=|ad_provider=/.test(target);
|
||||
}
|
||||
}
|
||||
|
||||
@singleton()
|
||||
export class StandaloneSearchFallbackService extends AsyncService {
|
||||
logger = this.globalLogger.child({ service: this.constructor.name });
|
||||
|
||||
constructor(
|
||||
protected globalLogger: GlobalLogger,
|
||||
protected jsDomControl: JSDomControl,
|
||||
) {
|
||||
super(...arguments);
|
||||
}
|
||||
|
||||
override async init() {
|
||||
await this.dependencyReady();
|
||||
this.emit('ready');
|
||||
}
|
||||
|
||||
async webSearch(query: { [k: string]: any; }) {
|
||||
const url = new URL('https://html.duckduckgo.com/html/');
|
||||
url.searchParams.set('q', query.q || '');
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36',
|
||||
'accept-language': query.hl || 'en-US,en;q=0.9',
|
||||
},
|
||||
signal: AbortSignal.timeout(20_000),
|
||||
}).catch((err) => {
|
||||
throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback failed: ${err instanceof Error ? err.message : err}`);
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback returned ${response.status}`);
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
const { document } = this.jsDomControl.linkedom.parseHTML(html).window;
|
||||
const nodes = Array.from(document.querySelectorAll('.result:not(.result--ad)'));
|
||||
const results = nodes.map((node) => {
|
||||
const linkElem = node.querySelector('.result__title a[href], .result__a[href]');
|
||||
const title = linkElem?.textContent?.replace(/\s+/g, ' ').trim();
|
||||
const href = linkElem?.getAttribute('href');
|
||||
|
||||
if (!title || !href) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const link = decodeDuckDuckGoTarget(href);
|
||||
if (isAdLikeDuckDuckGoTarget(link)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const snippet = node.querySelector('.result__snippet, .result-snippet')?.textContent?.replace(/\s+/g, ' ').trim();
|
||||
const source = node.querySelector('.result__url, .result__extras__url')?.textContent?.replace(/\s+/g, ' ').trim();
|
||||
|
||||
return {
|
||||
link,
|
||||
title,
|
||||
snippet: snippet || undefined,
|
||||
source: source || undefined,
|
||||
variant: 'web',
|
||||
} as WebSearchEntry;
|
||||
}).filter(Boolean) as WebSearchEntry[];
|
||||
|
||||
if (!results.length) {
|
||||
throw new ServiceBadAttemptError('DuckDuckGo HTML fallback returned no results.');
|
||||
}
|
||||
|
||||
return results.slice(0, query.num || 10);
|
||||
}
|
||||
|
||||
async newsSearch() {
|
||||
throw new ServiceBadApproachError('Standalone HTML fallback does not support news search.');
|
||||
}
|
||||
|
||||
async imageSearch() {
|
||||
throw new ServiceBadApproachError('Standalone HTML fallback does not support image search.');
|
||||
}
|
||||
}
|
||||
@@ -75,7 +75,7 @@ function applyPatch(base: any, patch: any) {
|
||||
}
|
||||
|
||||
class LocalStore {
|
||||
rootDir = path.resolve('.cache/xread/db');
|
||||
rootDir = path.resolve(process.env.LOCAL_DB_ROOT || '.cache/xread/db');
|
||||
|
||||
constructor() {
|
||||
fs.mkdirSync(this.rootDir, { recursive: true });
|
||||
|
||||
@@ -31,6 +31,10 @@ export class ProxyProviderService extends AsyncService {
|
||||
return /^[a-z]{2}$/i.test(input);
|
||||
}
|
||||
|
||||
isConfigured() {
|
||||
return this.proxies.length > 0;
|
||||
}
|
||||
|
||||
async alloc(input?: string) {
|
||||
if (input === 'none') {
|
||||
throw new ServiceDisabledError(`Proxy allocation is disabled for '${input}'.`);
|
||||
@@ -48,4 +52,3 @@ export class ProxyProviderService extends AsyncService {
|
||||
yield await this.alloc(input);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ type EnvConfig = Record<string, string> & {
|
||||
readonly CLOUD_FLARE_API_KEY: string;
|
||||
readonly LOCAL_PROXY_URLS: string;
|
||||
readonly STORAGE_ROOT: string;
|
||||
readonly LOCAL_DB_ROOT: string;
|
||||
};
|
||||
|
||||
export const readEnv = (key: string) => process.env[key] || '';
|
||||
@@ -43,6 +44,10 @@ export class SecretExposer extends AsyncService {
|
||||
get STORAGE_ROOT() {
|
||||
return readEnv('STORAGE_ROOT');
|
||||
}
|
||||
|
||||
get LOCAL_DB_ROOT() {
|
||||
return readEnv('LOCAL_DB_ROOT');
|
||||
}
|
||||
}
|
||||
|
||||
export default envConfig;
|
||||
@@ -0,0 +1,76 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
|
||||
const projectRoot = path.resolve(__dirname, '..');
|
||||
|
||||
function readProjectFile(relativePath) {
|
||||
return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8');
|
||||
}
|
||||
|
||||
test('browser launchers include container-safe Chrome sandbox flags', () => {
|
||||
const mainPuppeteer = readProjectFile('src/services/puppeteer.ts');
|
||||
const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts');
|
||||
|
||||
for (const source of [mainPuppeteer, serpPuppeteer]) {
|
||||
assert.match(source, /pipe:\s*true/);
|
||||
assert.match(source, /--no-sandbox/);
|
||||
assert.match(source, /--disable-setuid-sandbox/);
|
||||
}
|
||||
});
|
||||
|
||||
test('SERP puppeteer control uses an absolute control timeout instead of a fixed post-navigation race', () => {
|
||||
const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts');
|
||||
|
||||
assert.match(serpPuppeteer, /waitUntil:\s*\['domcontentloaded', 'load'\]/);
|
||||
assert.match(serpPuppeteer, /setTimeout\(\(\) => \{\s*resultDeferred\.reject\(new TimeoutError/);
|
||||
assert.doesNotMatch(serpPuppeteer, /networkidle0/);
|
||||
assert.doesNotMatch(serpPuppeteer, /await delay\(5000\)/);
|
||||
});
|
||||
|
||||
test('canvas service falls back when bundled font asset is missing', () => {
|
||||
const canvasService = readProjectFile('src/services/canvas.ts');
|
||||
|
||||
assert.match(canvasService, /ENOENT/);
|
||||
assert.match(canvasService, /Bundled font asset unavailable/);
|
||||
});
|
||||
|
||||
test('search host includes Google scraping fallback for standalone builds', () => {
|
||||
const searcherHost = readProjectFile('src/api/searcher.ts');
|
||||
|
||||
assert.match(searcherHost, /protected googleSerp: GoogleSERP/);
|
||||
assert.match(searcherHost, /protected standaloneFallback: StandaloneSearchFallbackService/);
|
||||
assert.match(searcherHost, /yield this\.googleSerp/);
|
||||
assert.match(searcherHost, /yield this\.standaloneFallback/);
|
||||
assert.match(searcherHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/);
|
||||
});
|
||||
|
||||
test('SERP host skips Serper providers when no API key is configured', () => {
|
||||
const serpHost = readProjectFile('src/api/serp.ts');
|
||||
|
||||
assert.match(serpHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/);
|
||||
assert.match(serpHost, /yield this\.standaloneFallback/);
|
||||
});
|
||||
|
||||
test('standalone fallback search provider is available for local web search without private APIs', () => {
|
||||
const fallbackProvider = readProjectFile('src/services/serp/standalone-fallback.ts');
|
||||
|
||||
assert.match(fallbackProvider, /DuckDuckGo HTML fallback/i);
|
||||
assert.match(fallbackProvider, /querySelectorAll\('\.result:not\(\.result--ad\)'\)/);
|
||||
assert.match(fallbackProvider, /ad_domain/);
|
||||
});
|
||||
|
||||
test('google search provider can fall back to direct requests when no proxy is configured', () => {
|
||||
const googleSerp = readProjectFile('src/services/serp/google.ts');
|
||||
|
||||
assert.match(googleSerp, /falling back to direct Google request/i);
|
||||
});
|
||||
|
||||
test('standalone URL validation can allow Docker DNS rewrite ranges behind an explicit env gate', () => {
|
||||
const miscService = readProjectFile('src/services/misc.ts');
|
||||
|
||||
assert.match(miscService, /STANDALONE_ALLOW_INTERNAL_DNS_REWRITE/);
|
||||
assert.match(miscService, /198\.18/);
|
||||
assert.match(miscService, /fd00::/i);
|
||||
});
|
||||
@@ -92,5 +92,8 @@ test('Dockerfile is self-contained and no longer relies on curl-impersonate', ()
|
||||
|
||||
assert.match(dockerfile, /FROM node:22-bookworm-slim/);
|
||||
assert.match(dockerfile, /PUPPETEER_SKIP_DOWNLOAD=true/);
|
||||
assert.match(dockerfile, /LOCAL_DB_ROOT=\/tmp\/xread\/db/);
|
||||
assert.match(dockerfile, /STORAGE_ROOT=\/tmp\/xread\/storage/);
|
||||
assert.doesNotMatch(dockerfile, /chown -R xread:xread \/app/);
|
||||
assert.doesNotMatch(dockerfile, /curl-impersonate|LD_PRELOAD/);
|
||||
});
|
||||
Reference in new issue
Block a user