feat(security): harden dependency governance

.github/workflows/ci.yml

@@ -40,6 +40,9 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      - name: Enforce dependency security policy
+        run: npm run security:audit
+
       - name: Lint source files
         run: npm run lint
 

.github/workflows/security-governance.yml

@@ -0,0 +1,45 @@
+name: Security Governance
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  schedule:
+    - cron: '31 17 * * 1'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+jobs:
+  npm-audit-policy:
+    name: Enforce npm audit policy
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: 22
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Enforce dependency security policy
+        run: npm run security:audit
+
+      - name: Upload security audit artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: security-audit-report
+          path: security-reports/
+          if-no-files-found: error

.gitignore

@@ -77,3 +77,4 @@ build/
 *.local
 .secret.*
 licensed/
+security-reports/

SECURITY.md

@@ -0,0 +1,16 @@
+# Security Policy
+
+## Supported Versions
+
+The supported release line is the latest `main` branch build and the container images published from it.
+
+## Reporting a Vulnerability
+
+Please open a private security advisory in GitHub when possible. If that is not available, open an issue with the minimum public detail needed to reproduce the problem and clearly mark it as security-sensitive.
+
+## Dependency Governance
+
+- Pull requests run `npm run security:audit`.
+- Blocking policy is based on `npm audit --omit=dev`: new runtime `high` or `critical` findings fail CI.
+- Development-only findings that are not yet practical to remove immediately must be recorded in `security/npm-audit-baseline.json` with a reason and review date.
+- Scheduled GitHub Actions runs refresh the audit report and upload artifacts under `security-reports/`.

package.json

@@ -2,7 +2,8 @@
   "name": "xread",
   "scripts": {
     "lint": "eslint --ext .js,.ts .",
-    "test:ci": "node --test tests/open-source-build.test.cjs tests/integrity-check.test.cjs tests/prepare-licensed-assets.test.cjs tests/github-automation.test.cjs tests/container-runtime-regressions.test.cjs",
+    "security:audit": "node ./scripts/security-audit-report.cjs",
+    "test:ci": "node --test tests/open-source-build.test.cjs tests/integrity-check.test.cjs tests/prepare-licensed-assets.test.cjs tests/github-automation.test.cjs tests/container-runtime-regressions.test.cjs tests/security-governance.test.cjs",
     "prepare:licensed-assets": "node ./scripts/prepare-licensed-assets.cjs",
     "build": "node ./integrity-check.cjs && tsc -p .",
     "build:watch": "tsc -p . -w",
@@ -18,7 +19,6 @@
   "main": "build/index.js",
   "dependencies": {
     "@esm2cjs/normalize-url": "^8.0.0",
-    "@google-cloud/translate": "^8.2.0",
     "@koa/bodyparser": "^5.1.1",
     "@mozilla/readability": "^0.6.0",
     "@napi-rs/canvas": "^0.1.97",
@@ -26,14 +26,11 @@
     "@xmldom/xmldom": "^0.9.9",
     "archiver": "^7.0.1",
     "axios": "^1.14.0",
-    "bcrypt": "^5.1.0",
+    "bcrypt": "^6.0.0",
     "busboy": "^1.6.0",
-    "civkit": "^0.9.0-2570394",
+    "civkit": "^0.9.1-a38f565",
     "cors": "^2.8.6",
     "dayjs": "^1.11.20",
-    "express": "^4.19.2",
-    "firebase-admin": "^12.1.0",
-    "firebase-functions": "^6.1.1",
     "htmlparser2": "^9.0.0",
     "jose": "^6.2.2",
     "koa": "^2.16.4",
@@ -42,7 +39,7 @@
     "linkedom": "^0.18.12",
     "lru-cache": "^11.2.7",
     "maxmind": "^4.3.18",
-    "minio": "^7.1.3",
+    "minio": "^8.0.7",
     "openai": "^4.20.0",
     "pdfjs-dist": "^4.10.38",
     "puppeteer": "^23.3.0",
@@ -66,6 +63,7 @@
     "@types/cors": "^2.8.19",
     "@types/koa": "^2.15.0",
     "@types/koa-compress": "^4.0.6",
+    "@types/lodash": "^4.17.24",
     "@types/node": "^20.14.13",
     "@types/set-cookie-parser": "^2.4.7",
     "@types/xmldom": "^0.1.34",
@@ -74,11 +72,28 @@
     "eslint": "^8.9.0",
     "eslint-config-google": "^0.14.0",
     "eslint-plugin-import": "^2.25.4",
-    "firebase-functions-test": "^3.0.0",
     "pino-pretty": "^13.0.0",
     "replicate": "^0.16.1",
     "typescript": "^5.5.4"
   },
+  "overrides": {
+    "ajv": "^6.14.0",
+    "express": "^4.22.1",
+    "body-parser": "^1.20.4",
+    "qs": "^6.15.0",
+    "path-to-regexp": "^0.1.13",
+    "minimatch": "^3.1.5",
+    "minio": "^8.0.7",
+    "fast-xml-parser": "^5.5.9",
+    "nodemailer": "^8.0.4",
+    "smtp-server": "^3.18.3",
+    "readdir-glob": {
+      "minimatch": "^5.1.9"
+    },
+    "civkit": {
+      "minimatch": "^10.2.4"
+    }
+  },
   "private": true,
   "exports": {
     ".": "./build/index.js"

scripts/security-audit-report.cjs

@@ -0,0 +1,198 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+
+const projectRoot = path.resolve(__dirname, '..');
+const baselinePath = path.join(projectRoot, 'security', 'npm-audit-baseline.json');
+const reportDirectory = path.join(projectRoot, 'security-reports');
+const summaryJsonPath = path.join(reportDirectory, 'npm-audit-summary.json');
+const summaryMarkdownPath = path.join(reportDirectory, 'npm-audit-summary.md');
+
+const severityRank = {
+  info: 0,
+  low: 1,
+  moderate: 2,
+  high: 3,
+  critical: 4,
+};
+
+function npmCommand() {
+  return process.platform === 'win32' ? 'npm.cmd' : 'npm';
+}
+
+function readBaseline() {
+  const raw = fs.readFileSync(baselinePath, 'utf8');
+  const parsed = JSON.parse(raw);
+  return Array.isArray(parsed.entries) ? parsed.entries : [];
+}
+
+function runAudit(extraArgs) {
+  const result = spawnSync(npmCommand(), ['audit', '--json', ...extraArgs], {
+    cwd: projectRoot,
+    encoding: 'utf8',
+    env: process.env,
+  });
+
+  if (![0, 1].includes(result.status ?? 0)) {
+    process.stderr.write(result.stderr || result.stdout || 'npm audit failed.\n');
+    process.exit(result.status ?? 1);
+  }
+
+  const combinedOutput = `${result.stdout || ''}\n${result.stderr || ''}`.trim();
+  const jsonStart = combinedOutput.indexOf('{');
+  if (jsonStart === -1) {
+    if ((result.status ?? 0) === 0) {
+      return {
+        auditReportVersion: 2,
+        vulnerabilities: {},
+        metadata: {
+          vulnerabilities: {
+            info: 0,
+            low: 0,
+            moderate: 0,
+            high: 0,
+            critical: 0,
+            total: 0,
+          },
+        },
+      };
+    }
+
+    throw new Error('npm audit returned no JSON payload.');
+  }
+
+  return JSON.parse(combinedOutput.slice(jsonStart));
+}
+
+function toFindings(report, scope) {
+  return Object.entries(report.vulnerabilities || {}).map(([pkg, entry]) => {
+    const advisories = (entry.via || [])
+      .filter((item) => item && typeof item === 'object' && item.title)
+      .map((item) => ({
+        source: item.source,
+        title: item.title,
+        severity: item.severity,
+        url: item.url,
+      }));
+
+    return {
+      package: pkg,
+      scope,
+      severity: entry.severity,
+      isDirect: Boolean(entry.isDirect),
+      nodes: entry.nodes || [],
+      effects: entry.effects || [],
+      advisories,
+    };
+  });
+}
+
+function matchBaseline(finding, baselineEntries) {
+  return baselineEntries.find((entry) =>
+    entry.package === finding.package &&
+    entry.scope === finding.scope &&
+    entry.severity === finding.severity
+  );
+}
+
+function formatFinding(finding, baselineEntry) {
+  const headline = `${finding.package} (${finding.scope}, ${finding.severity})`;
+  const notes = [];
+
+  if (finding.advisories.length) {
+    notes.push(`advisories: ${finding.advisories.map((item) => item.title).join(' | ')}`);
+  }
+  if (finding.effects.length) {
+    notes.push(`effects: ${finding.effects.join(', ')}`);
+  }
+  if (finding.nodes.length) {
+    notes.push(`nodes: ${finding.nodes.join(', ')}`);
+  }
+  if (baselineEntry) {
+    notes.push(`baseline: ${baselineEntry.reason} (review by ${baselineEntry.reviewBy})`);
+  }
+
+  return `- ${headline}\n  ${notes.join('\n  ')}`;
+}
+
+function writeReports(payload) {
+  fs.mkdirSync(reportDirectory, { recursive: true });
+  fs.writeFileSync(summaryJsonPath, JSON.stringify(payload, null, 2) + '\n');
+
+  const sections = [
+    '# npm audit summary',
+    '',
+    `- generatedAt: ${payload.generatedAt}`,
+    `- blockingFindings: ${payload.blockingFindings.length}`,
+    `- baselinedDevelopmentFindings: ${payload.baselinedDevelopmentFindings.length}`,
+    `- productionVulnerabilities: ${JSON.stringify(payload.production.metadata.vulnerabilities)}`,
+    `- fullAuditVulnerabilities: ${JSON.stringify(payload.full.metadata.vulnerabilities)}`,
+    '',
+    '## Blocking findings',
+    ...(payload.blockingFindings.length
+      ? payload.blockingFindings.map((item) => formatFinding(item, null))
+      : ['- none']),
+    '',
+    '## Baselined development findings',
+    ...(payload.baselinedDevelopmentFindings.length
+      ? payload.baselinedDevelopmentFindings.map((item) => formatFinding(item.finding, item.baseline))
+      : ['- none']),
+  ];
+
+  fs.writeFileSync(summaryMarkdownPath, sections.join('\n') + '\n');
+}
+
+const baselineEntries = readBaseline();
+const productionAudit = runAudit(['--omit=dev']);
+const fullAudit = runAudit([]);
+
+const productionFindings = toFindings(productionAudit, 'production');
+const fullFindings = toFindings(fullAudit, 'development');
+
+const baselinedDevelopmentFindings = [];
+const blockingFindings = [];
+
+for (const finding of productionFindings) {
+  if (severityRank[finding.severity] >= severityRank.high) {
+    blockingFindings.push(finding);
+  }
+}
+
+for (const finding of fullFindings) {
+  if (productionFindings.some((prodFinding) => prodFinding.package === finding.package)) {
+    continue;
+  }
+
+  const baselineEntry = matchBaseline(finding, baselineEntries);
+  if (baselineEntry) {
+    baselinedDevelopmentFindings.push({ finding, baseline: baselineEntry });
+    continue;
+  }
+
+  if (severityRank[finding.severity] >= severityRank.high) {
+    blockingFindings.push(finding);
+  }
+}
+
+const payload = {
+  generatedAt: new Date().toISOString(),
+  production: {
+    metadata: productionAudit.metadata,
+    findings: productionFindings,
+  },
+  full: {
+    metadata: fullAudit.metadata,
+    findings: fullFindings,
+  },
+  blockingFindings,
+  baselinedDevelopmentFindings,
+};
+
+writeReports(payload);
+
+if (blockingFindings.length) {
+  process.stderr.write(`Security audit failed with ${blockingFindings.length} blocking finding(s).\n`);
+  process.exit(1);
+}
+
+process.stdout.write('Security audit policy passed.\n');

security/npm-audit-baseline.json

@@ -0,0 +1,3 @@
+{
+  "entries": []
+}

src/dto/adaptive-crawler-options.ts

@@ -1,5 +1,8 @@
 import { Also, AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from 'civkit';
-import type { Request, Response } from 'express';
+
+type HeaderReadableRequest = {
+    get(name: string): string | undefined;
+};
 
 
 @Also({
@@ -42,8 +45,7 @@ export class AdaptiveCrawlerOptions extends AutoCastable {
     static override from(input: any) {
         const instance = super.from(input) as AdaptiveCrawlerOptions;
         const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {
-            req: Request,
-            res: Response,
+            req: HeaderReadableRequest,
         } | undefined;
 
         let maxPages = parseInt(ctx?.req.get('x-max-pages') || '');

src/services/brave-search.ts

@@ -6,9 +6,12 @@ import { BraveSearchHTTP, WebSearchQueryParams } from '../shared/3rd-party/brave
 import { GEOIP_SUPPORTED_LANGUAGES, GeoIPService } from './geoip';
 import { AsyncLocalContext } from './async-context';
 import { WebSearchOptionalHeaderOptions } from '../shared/3rd-party/brave-types';
-import type { Request, Response } from 'express';
 import { BlackHoleDetector } from './blackhole-detector';
 
+type HeaderReadableRequest = {
+    get(name: string): string | undefined;
+};
+
 @singleton()
 export class BraveSearchService extends AsyncService {
 
@@ -163,8 +166,7 @@ export class BraveSearchExplicitOperatorsDto extends AutoCastable {
     static override from(input: any) {
         const instance = super.from(input) as BraveSearchExplicitOperatorsDto;
         const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as {
-            req: Request,
-            res: Response,
+            req: HeaderReadableRequest,
         } | undefined;
 
         const params = ['ext', 'filetype', 'inbody', 'intitle', 'inpage', 'lang', 'loc', 'site'];

tests/security-governance.test.cjs

@@ -0,0 +1,38 @@
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const projectRoot = path.resolve(__dirname, '..');
+
+function read(relativePath) {
+  return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8');
+}
+
+test('package.json exposes the security audit script and excludes removed cloud dependencies', () => {
+  const packageJson = JSON.parse(read('package.json'));
+
+  assert.equal(packageJson.scripts['security:audit'], 'node ./scripts/security-audit-report.cjs');
+  assert.match(packageJson.scripts['test:ci'], /tests\/security-governance\.test\.cjs/);
+  assert.ok(!('@google-cloud/translate' in packageJson.dependencies));
+  assert.ok(!('express' in packageJson.dependencies));
+  assert.ok(!('firebase-admin' in packageJson.dependencies));
+  assert.ok(!('firebase-functions' in packageJson.dependencies));
+  assert.ok(!('firebase-functions-test' in packageJson.devDependencies));
+});
+
+test('security governance workflow enforces npm audit policy and uploads reports', () => {
+  const workflow = read('.github/workflows/security-governance.yml');
+
+  assert.match(workflow, /name:\s+Security Governance/);
+  assert.match(workflow, /run:\s+npm run security:audit/);
+  assert.match(workflow, /actions\/upload-artifact@v4/);
+  assert.match(workflow, /cron:\s+'31 17 \* \* 1'/);
+});
+
+test('security baseline file is present and ready for future exceptions', () => {
+  const baseline = JSON.parse(read('security/npm-audit-baseline.json'));
+
+  assert.ok(Array.isArray(baseline.entries));
+  assert.equal(baseline.entries.length, 0);
+});

tsconfig.json

@@ -19,5 +19,5 @@
     },
     "compileOnSave": true,
     "include": ["src"],
-    "exclude": ["src/cloud-functions/**/*"]
+    "exclude": ["src/cloud-functions/**/*", "src/utils/get-function-url.ts"]
 }