diff --git a/package.json b/package.json index 632956b..cdb7657 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "xread", "scripts": { "lint": "eslint --ext .js,.ts .", - "test:ci": "node --test tests/open-source-build.test.cjs tests/integrity-check.test.cjs tests/prepare-licensed-assets.test.cjs tests/github-automation.test.cjs", + "test:ci": "node --test tests/open-source-build.test.cjs tests/integrity-check.test.cjs tests/prepare-licensed-assets.test.cjs tests/github-automation.test.cjs tests/container-runtime-regressions.test.cjs", "prepare:licensed-assets": "node ./scripts/prepare-licensed-assets.cjs", "build": "node ./integrity-check.cjs && tsc -p .", "build:watch": "tsc -p . -w", diff --git a/src/api/crawler.ts b/src/api/crawler.ts index 992d723..6a80ac8 100644 --- a/src/api/crawler.ts +++ b/src/api/crawler.ts @@ -29,6 +29,7 @@ import { CurlControl } from '../services/curl'; import { LmControl } from '../services/lm'; import { tryDecodeURIComponent } from '../utils/misc'; import { CFBrowserRendering } from '../services/cf-browser-rendering'; +import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; import { GlobalLogger } from '../services/logger'; import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit'; @@ -178,7 +179,7 @@ export class CrawlerHost extends RPCHost { } override async init() { - await this.dependencyReady(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); if (this.puppeteerControl.effectiveUA) { this.curlControl.impersonateChrome(this.puppeteerControl.effectiveUA); diff --git a/src/api/searcher.ts b/src/api/searcher.ts index 3f51c43..5e54ec1 100644 --- a/src/api/searcher.ts +++ b/src/api/searcher.ts @@ -29,6 +29,7 @@ import { SerperSearchQueryParams } from '../shared/3rd-party/serper-search'; import { WebSearchEntry } from '../services/serp/compat'; import { SecretExposer } from '../shared/services/secrets'; import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback'; +import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; interface FormattedPage extends RealFormattedPage { favicon?: string; @@ -97,7 +98,7 @@ export class SearcherHost extends RPCHost { } override async init() { - await this.dependencyReady(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); this.emit('ready'); } diff --git a/src/api/serp.ts b/src/api/serp.ts index 2005bf6..7ac18c6 100644 --- a/src/api/serp.ts +++ b/src/api/serp.ts @@ -26,6 +26,7 @@ import { LRUCache } from 'lru-cache'; import { API_CALL_STATUS } from '../shared/db/api-roll'; import { SecretExposer } from '../shared/services/secrets'; import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback'; +import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; type RateLimitCache = { blockedUntil?: Date; @@ -115,7 +116,7 @@ export class SerpHost extends RPCHost { } override async init() { - await this.dependencyReady(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); this.emit('ready'); } diff --git a/src/services/boot-timeouts.ts b/src/services/boot-timeouts.ts new file mode 100644 index 0000000..4762cde --- /dev/null +++ b/src/services/boot-timeouts.ts @@ -0,0 +1,2 @@ +export const STANDALONE_BOOT_TIMEOUT_MS = 90_000; +export const SERP_BOOT_TIMEOUT_MS = 90_000; diff --git a/src/services/serp/google.ts b/src/services/serp/google.ts index 25e6af5..fea6fa2 100644 --- a/src/services/serp/google.ts +++ b/src/services/serp/google.ts @@ -13,6 +13,7 @@ import { ServiceBadApproachError, ServiceBadAttemptError } from '../errors'; import { parseJSONText } from 'civkit/vectorize'; import { retryWith } from 'civkit/decorators'; import { ProxyProviderService } from '../../shared/services/proxy-provider'; +import { SERP_BOOT_TIMEOUT_MS } from '../boot-timeouts'; @singleton() export class GoogleSERP extends AsyncService { @@ -31,7 +32,7 @@ export class GoogleSERP extends AsyncService { } override async init() { - await this.dependencyReady(); + await this.dependencyReady(SERP_BOOT_TIMEOUT_MS); this.emit('ready'); } diff --git a/src/stand-alone/crawl.ts b/src/stand-alone/crawl.ts index 4a6e84c..34d3906 100644 --- a/src/stand-alone/crawl.ts +++ b/src/stand-alone/crawl.ts @@ -19,6 +19,7 @@ import { GlobalLogger } from '../services/logger'; import { AsyncLocalContext } from '../services/async-context'; import finalizer, { Finalizer } from '../services/finalizer'; import koaCompress from 'koa-compress'; +import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; @singleton() export class CrawlStandAloneServer extends KoaServer { @@ -51,6 +52,7 @@ export class CrawlStandAloneServer extends KoaServer { override async init() { await this.walkForAssets(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); await super.init(); } @@ -166,4 +168,4 @@ if (process.env.NODE_ENV?.includes('dry-run')) { instance.serviceReady().then(() => finalizer.terminate()); } else { instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000)); -} \ No newline at end of file +} diff --git a/src/stand-alone/search.ts b/src/stand-alone/search.ts index 673a3cb..33de155 100644 --- a/src/stand-alone/search.ts +++ b/src/stand-alone/search.ts @@ -19,6 +19,7 @@ import { GlobalLogger } from '../services/logger'; import { AsyncLocalContext } from '../services/async-context'; import finalizer, { Finalizer } from '../services/finalizer'; import koaCompress from 'koa-compress'; +import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; @singleton() export class SearchStandAloneServer extends KoaServer { @@ -51,7 +52,7 @@ export class SearchStandAloneServer extends KoaServer { override async init() { await this.walkForAssets(); - await this.dependencyReady(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); for (const [k, v] of this.registry.conf.entries()) { if (v.tags?.includes('crawl')) { diff --git a/src/stand-alone/serp.ts b/src/stand-alone/serp.ts index 76c91fc..c5a19dc 100644 --- a/src/stand-alone/serp.ts +++ b/src/stand-alone/serp.ts @@ -20,6 +20,7 @@ import finalizer, { Finalizer } from '../services/finalizer'; import { SerpHost } from '../api/serp'; import koaCompress from 'koa-compress'; import { getAuditionMiddleware } from '../shared/utils/audition'; +import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; @singleton() export class SERPStandAloneServer extends KoaServer { @@ -52,7 +53,7 @@ export class SERPStandAloneServer extends KoaServer { override async init() { await this.walkForAssets(); - await this.dependencyReady(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); for (const [k, v] of this.registry.conf.entries()) { if (v.tags?.includes('crawl')) { diff --git a/tests/container-runtime-regressions.test.cjs b/tests/container-runtime-regressions.test.cjs index 8af92b3..a0c0057 100644 --- a/tests/container-runtime-regressions.test.cjs +++ b/tests/container-runtime-regressions.test.cjs @@ -74,3 +74,24 @@ test('standalone URL validation can allow Docker DNS rewrite ranges behind an ex assert.match(miscService, /198\.18/); assert.match(miscService, /fd00::/i); }); + +test('standalone services use extended boot timeouts for cold starts', () => { + const bootTimeouts = readProjectFile('src/services/boot-timeouts.ts'); + const crawlHost = readProjectFile('src/api/crawler.ts'); + const searcherHost = readProjectFile('src/api/searcher.ts'); + const serpHost = readProjectFile('src/api/serp.ts'); + const crawlServer = readProjectFile('src/stand-alone/crawl.ts'); + const searchServer = readProjectFile('src/stand-alone/search.ts'); + const serpServer = readProjectFile('src/stand-alone/serp.ts'); + const googleSerp = readProjectFile('src/services/serp/google.ts'); + + assert.match(bootTimeouts, /STANDALONE_BOOT_TIMEOUT_MS = 90_000/); + assert.match(bootTimeouts, /SERP_BOOT_TIMEOUT_MS = 90_000/); + assert.match(crawlHost, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match(searcherHost, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match(serpHost, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match(crawlServer, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match(searchServer, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match(serpServer, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match(googleSerp, /dependencyReady\(SERP_BOOT_TIMEOUT_MS\)/); +});