fix(runtime): harden standalone storage and search
This commit is contained in:
1 parent
bc26578f0a
commit
0bd47ad1fb
3 files changed
+72
-7
No files matched your search
+6
-4
@@ -64,7 +64,8 @@ export class SearcherHost extends RPCHost {
|
||||
cacheValidMs = 1000 * 3600;
|
||||
pageCacheToleranceMs = 1000 * 3600 * 24;
|
||||
|
||||
reasonableDelayMs = 15_000;
|
||||
reasonableDelayMs = 8_000;
|
||||
standaloneSearchPageTimeoutMs = 12_000;
|
||||
|
||||
targetResultCount = 5;
|
||||
|
||||
@@ -394,9 +395,10 @@ export class SearcherHost extends RPCHost {
|
||||
);
|
||||
}
|
||||
|
||||
if (crawlOpts.timeoutMs && crawlOpts.timeoutMs < 30_000) {
|
||||
delete crawlOpts.timeoutMs;
|
||||
}
|
||||
crawlOpts.timeoutMs = Math.min(
|
||||
crawlOpts.timeoutMs || this.standaloneSearchPageTimeoutMs,
|
||||
this.standaloneSearchPageTimeoutMs,
|
||||
);
|
||||
|
||||
let lastScrapped: any[] | undefined;
|
||||
const targetResultCount = crawlWithoutContent ? count : count + 2;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { AutoCastable } from "civkit/civ-rpc";
|
||||
import { randomUUID } from "crypto";
|
||||
import fs from "fs";
|
||||
import fsp from "fs/promises";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
|
||||
type WhereOp = "==" | ">=" | "<=" | ">" | "<";
|
||||
@@ -83,12 +84,46 @@ function applyPatch(base: any, patch: any) {
|
||||
}
|
||||
|
||||
class LocalStore {
|
||||
rootDir = path.resolve(process.env.LOCAL_DB_ROOT || ".cache/xread/db");
|
||||
rootDir = this.resolveWritableRoot(
|
||||
process.env.LOCAL_DB_ROOT || ".cache/xread/db",
|
||||
);
|
||||
|
||||
constructor() {
|
||||
fs.mkdirSync(this.rootDir, { recursive: true });
|
||||
}
|
||||
|
||||
protected resolveWritableRoot(preferredRoot: string) {
|
||||
const candidates = [
|
||||
path.resolve(preferredRoot),
|
||||
path.join(os.tmpdir(), "xread", "db"),
|
||||
];
|
||||
let lastError;
|
||||
|
||||
for (const candidate of [...new Set(candidates)]) {
|
||||
try {
|
||||
fs.mkdirSync(candidate, { recursive: true });
|
||||
const probe = path.join(
|
||||
candidate,
|
||||
`.write-test-${process.pid}-${randomUUID()}`,
|
||||
);
|
||||
fs.writeFileSync(probe, "");
|
||||
fs.rmSync(probe, { force: true });
|
||||
|
||||
if (candidate !== path.resolve(preferredRoot)) {
|
||||
console.warn(
|
||||
`[xread] Local DB root ${path.resolve(preferredRoot)} is not writable, falling back to ${candidate}`,
|
||||
);
|
||||
}
|
||||
|
||||
return candidate;
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
filePath(collectionName: string) {
|
||||
return path.join(this.rootDir, `${collectionName}.json`);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { AsyncService } from "civkit/async-service";
|
||||
import { singleton } from "tsyringe";
|
||||
import os from "os";
|
||||
import path from "path";
|
||||
import fs from "fs/promises";
|
||||
import { pathToFileURL } from "url";
|
||||
@@ -22,13 +23,40 @@ export class FirebaseStorageBucketControl extends AsyncService {
|
||||
|
||||
override async init() {
|
||||
await this.dependencyReady();
|
||||
this.rootDir = path.resolve(
|
||||
this.rootDir = await this.resolveWritableRoot(
|
||||
this.secretExposer.STORAGE_ROOT || ".cache/xread/storage",
|
||||
);
|
||||
await fs.mkdir(this.rootDir, { recursive: true });
|
||||
this.emit("ready");
|
||||
}
|
||||
|
||||
protected async resolveWritableRoot(preferredRoot: string) {
|
||||
const preferred = path.resolve(preferredRoot);
|
||||
const fallback = path.join(os.tmpdir(), "xread", "storage");
|
||||
|
||||
for (const candidate of [preferred, fallback]) {
|
||||
try {
|
||||
await fs.mkdir(candidate, { recursive: true });
|
||||
const probe = path.join(candidate, `.write-test-${process.pid}`);
|
||||
await fs.writeFile(probe, "");
|
||||
await fs.rm(probe, { force: true });
|
||||
|
||||
if (candidate !== preferred) {
|
||||
console.warn(
|
||||
`[xread] Storage root ${preferred} is not writable, falling back to ${candidate}`,
|
||||
);
|
||||
}
|
||||
|
||||
return candidate;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Neither preferred nor fallback storage roots are writable: ${preferred}`,
|
||||
);
|
||||
}
|
||||
|
||||
protected resolvePath(key: string) {
|
||||
const safeKey = key.replace(/\\/g, "/").replace(/^\/+/, "");
|
||||
return path.resolve(this.rootDir, safeKey);
|
||||
|
||||
Reference in new issue
Block a user