diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bbcb8c0..9dd265c 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -8,7 +8,7 @@ on: branches: - main schedule: - - cron: '31 16 * * 0' + - cron: "31 16 * * 0" workflow_dispatch: permissions: diff --git a/.github/workflows/security-governance.yml b/.github/workflows/security-governance.yml index 54bc407..5f4f276 100644 --- a/.github/workflows/security-governance.yml +++ b/.github/workflows/security-governance.yml @@ -6,7 +6,7 @@ on: branches: - main schedule: - - cron: '31 17 * * 1' + - cron: "31 17 * * 1" workflow_dispatch: permissions: diff --git a/.vscode/exensions.json b/.vscode/exensions.json index 37bdb37..0ca6c01 100644 --- a/.vscode/exensions.json +++ b/.vscode/exensions.json @@ -1,10 +1,10 @@ { - "recommendations": [ - "editorconfig.editorconfig", - "octref.vetur", - "redhat.vscode-yaml", - "dbaeumer.vscode-eslint", - "esbenp.prettier-vscode", - "streetsidesoftware.code-spell-checker" - ] -} \ No newline at end of file + "recommendations": [ + "editorconfig.editorconfig", + "octref.vetur", + "redhat.vscode-yaml", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "streetsidesoftware.code-spell-checker" + ] +} diff --git a/.vscode/launch.json b/.vscode/launch.json index 2eacf75..38440ba 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,31 +5,23 @@ "name": "Attach", "port": 9229, "request": "attach", - "skipFiles": [ - "/**" - ], + "skipFiles": ["/**"], "type": "node" }, { "name": "Attach by Process ID", "processId": "${command:PickProcess}", "request": "attach", - "skipFiles": [ - "/**" - ], + "skipFiles": ["/**"], "type": "node" }, { "name": "Debug Stand Alone Crawl", "request": "launch", - "runtimeArgs": [ - "--env-file=.secret.local" - ], + "runtimeArgs": ["--env-file=.secret.local"], "cwd": "${workspaceFolder}", "program": "build/stand-alone/crawl.js", - "skipFiles": [ - "/**" - ], + "skipFiles": ["/**"], "type": "node", "outputCapture": "std", "preLaunchTask": "Backend:build:watch", @@ -38,17 +30,13 @@ { "name": "Debug Stand Alone Crawl + Browser", "request": "launch", - "runtimeArgs": [ - "--env-file=.secret.local" - ], + "runtimeArgs": ["--env-file=.secret.local"], "env": { "DEBUG_BROWSER": "true" }, "cwd": "${workspaceFolder}", "program": "build/stand-alone/crawl.js", - "skipFiles": [ - "/**" - ], + "skipFiles": ["/**"], "type": "node", "outputCapture": "std", "preLaunchTask": "Backend:build:watch", @@ -57,14 +45,10 @@ { "name": "Debug Stand Alone Search", "request": "launch", - "runtimeArgs": [ - "--env-file=.secret.local" - ], + "runtimeArgs": ["--env-file=.secret.local"], "cwd": "${workspaceFolder}", "program": "build/stand-alone/search.js", - "skipFiles": [ - "/**" - ], + "skipFiles": ["/**"], "type": "node", "outputCapture": "std", "preLaunchTask": "Backend:build:watch", @@ -73,14 +57,10 @@ { "name": "Debug Stand Alone SERP", "request": "launch", - "runtimeArgs": [ - "--env-file=.secret.local" - ], + "runtimeArgs": ["--env-file=.secret.local"], "cwd": "${workspaceFolder}", "program": "build/stand-alone/serp.js", - "skipFiles": [ - "/**" - ], + "skipFiles": ["/**"], "type": "node", "outputCapture": "std", "preLaunchTask": "Backend:build:watch", diff --git a/.vscode/settings.json b/.vscode/settings.json index 87ecca7..6e8a3b3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,32 +1,31 @@ { - "editor.wordWrap": "on", - "editor.wordWrapColumn": 120, - "files.trimTrailingWhitespace": true, - "files.trimFinalNewlines": true, - "[javascript]": { - "editor.defaultFormatter": "vscode.typescript-language-features" - }, - "[jsonc]": { - "editor.defaultFormatter": "vscode.json-language-features" - }, - "[typescript]": { - "editor.defaultFormatter": "vscode.typescript-language-features" - }, - "[json]": { - "editor.defaultFormatter": "vscode.json-language-features" - }, - "[yaml]": { - "editor.defaultFormatter": "redhat.vscode-yaml" - }, - "[markdown]": { - "files.trimTrailingWhitespace": false - }, - "typescript.tsdk": "node_modules/typescript/lib", - "typescript.preferences.quoteStyle": "single", - "typescript.format.semicolons": "insert", - "typescript.preferences.importModuleSpecifier": "project-relative", - "typescript.locale": "en", - "cSpell.enabled": true, - "cSpell.words": [ - ], -} \ No newline at end of file + "editor.wordWrap": "on", + "editor.wordWrapColumn": 120, + "files.trimTrailingWhitespace": true, + "files.trimFinalNewlines": true, + "[javascript]": { + "editor.defaultFormatter": "vscode.typescript-language-features" + }, + "[jsonc]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[typescript]": { + "editor.defaultFormatter": "vscode.typescript-language-features" + }, + "[json]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[yaml]": { + "editor.defaultFormatter": "redhat.vscode-yaml" + }, + "[markdown]": { + "files.trimTrailingWhitespace": false + }, + "typescript.tsdk": "node_modules/typescript/lib", + "typescript.preferences.quoteStyle": "single", + "typescript.format.semicolons": "insert", + "typescript.preferences.importModuleSpecifier": "project-relative", + "typescript.locale": "en", + "cSpell.enabled": true, + "cSpell.words": [] +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 5cba448..0da167b 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,30 +1,28 @@ { - "version": "2.0.0", - "tasks": [ - { - "type": "npm", - "script": "build", - "group": "build", - "options": { - "cwd": "${workspaceFolder}" - }, - "problemMatcher": [], - "label": "Backend:rebuild", - "detail": "Backend:rebuild" - }, - { - "type": "typescript", - "options": { - "cwd": "${workspaceFolder}" - }, - "tsconfig": "tsconfig.json", - "option": "watch", - "isBackground": true, - "problemMatcher": [ - "$tsc-watch" - ], - "group": "build", - "label": "Backend:build:watch" - } - ] -} \ No newline at end of file + "version": "2.0.0", + "tasks": [ + { + "type": "npm", + "script": "build", + "group": "build", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "label": "Backend:rebuild", + "detail": "Backend:rebuild" + }, + { + "type": "typescript", + "options": { + "cwd": "${workspaceFolder}" + }, + "tsconfig": "tsconfig.json", + "option": "watch", + "isBackground": true, + "problemMatcher": ["$tsc-watch"], + "group": "build", + "label": "Backend:build:watch" + } + ] +} diff --git a/README.md b/README.md index eb22a2e..f9ad792 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,161 @@ # Xread -Your LLMs deserve better input. +**_[汉语](./README.zh.md)_** -Xread does two things: -- **Read**: Convert any URL into an **LLM-friendly** representation with `https://r.example.com/https://your.url`. -- **Search**: Search the web with `https://s.example.com/your+query` and return summarized results in an LLM-friendly format. +**Your LLMs deserve better input.** -Check out the placeholder demo at [https://example.com/xread#demo](https://example.com/xread#demo). +Turn messy webpages into clean Markdown, run web search for LLMs, and self-host the whole stack. -## Usage +Xread is an open-source, self-hosted web ingestion service for LLM workflows. It gives you three main surfaces: -### Read a single URL +- `crawl`: fetch a URL and turn it into readable Markdown +- `search`: run a search and return text that is easy to paste into a prompt +- `serp`: run a search and return structured JSON -Prepend `https://r.example.com/` to any URL: +It is a standalone fork of the public [Jina AI Reader](https://jina.ai/reader) repository in [jina-ai/reader](https://github.com/jina-ai/reader), adapted for self-hosted use without the original internal `thinapps-shared` dependency. -[https://r.example.com/https://en.wikipedia.org/wiki/Artificial_intelligence](https://r.example.com/https://en.wikipedia.org/wiki/Artificial_intelligence) +## What This Repository Adds -### Search the web +- Self-hosted packaging around the upstream Reader repository +- Local-first storage and cache persistence +- OCI image published to GHCR +- Ready-to-run reverse-proxy deployment assets with the same image, Compose stack, and environment model across Docker, Podman, and `nerdctl` +- CI, CodeQL, dependency governance, and container publishing already wired up -Prepend `https://s.example.com/` to a URL-encoded search query: +## Quick Look -[https://s.example.com/Who%20will%20win%202024%20US%20presidential%20election%3F](https://s.example.com/Who%20will%20win%202024%20US%20presidential%20election%3F) +These examples use the published image directly on its HTTP/1 port (`8081`). The deployment guide later covers the Compose backend loopback ports (`3001`, `3101`, `3201`) and the public reverse-proxy hostnames. -Behind the scenes, Xread fetches relevant pages and converts them into a format that is easier for downstream LLMs and agent systems to consume. - -### In-site search - -Use repeated `site` parameters to constrain results: +### Crawl a page ```bash -curl 'https://s.example.com/When%20was%20example.com%20founded%3F?site=example.com&site=github.com' +curl "http://127.0.0.1:8081/http://example.com" ``` -### Interactive code builder - -Use the placeholder builder URL: - -[https://example.com/xread#apiform](https://example.com/xread#apiform) - -## Request headers - -The service behavior can be controlled via headers: - -- `x-with-generated-alt: true` enables automatic image alt-text generation. -- `x-set-cookie` forwards cookies. -- `x-respond-with` supports `markdown`, `html`, `text`, `screenshot`, and related formats. -- `x-proxy-url` selects a custom proxy. -- `x-cache-tolerance` adjusts cache tolerance in seconds. -- `x-no-cache: true` bypasses cached content. -- `x-target-selector` narrows extraction to a CSS selector. -- `x-wait-for-selector` waits until a CSS selector appears. - -## SPA fetching - -For hash-based routing, use `POST` with the target URL in the body: +### Search the web as text ```bash -curl -X POST 'https://r.example.com/' -d 'url=https://example.com/#/route' +curl "http://127.0.0.1:8081/search?q=example%20domain&num=5&provider=google" ``` -## Streaming mode +### Search the web as JSON ```bash -curl -H "Accept: text/event-stream" https://r.example.com/https://en.m.wikipedia.org/wiki/Main_Page +curl "http://127.0.0.1:8081/?q=example%20domain&num=5&provider=google" ``` -Streaming responses return progressively more complete content chunks. +## Quick Start -## JSON mode +### Run locally ```bash -curl -H "Accept: application/json" https://r.example.com/https://en.m.wikipedia.org/wiki/Main_Page +npm ci +npm run build +npm start ``` -For `s.example.com`, JSON mode returns a list of search results shaped like `{'title', 'content', 'url'}`. +That starts the `crawl` server. The standalone services are: -## Generated alt +- `build/stand-alone/crawl.js` +- `build/stand-alone/search.js` +- `build/stand-alone/serp.js` + +### Run with a container engine ```bash -curl -H "X-With-Generated-Alt: true" https://r.example.com/https://en.m.wikipedia.org/wiki/Main_Page +docker run --rm -p 8081:8081 ghcr.io/xixu-me/xread:latest ``` -## Standalone Notes +The published image defaults to `crawl`. Any OCI-compatible engine can run it. The examples below use Docker syntax. To run the other entrypoints: -This repository now carries its shared infrastructure helpers in `src/shared/` so the stand-alone crawl/search/serp entrypoints can build and run without private dependencies. +```bash +docker run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/search.js +docker run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/serp.js +``` -## License +## Built for Self-Hosting -This repository is distributed under [Apache-2.0](./LICENSE). See [NOTICE](./NOTICE) for modification and attribution details. +- Chrome is bundled in the image for page rendering +- HTTP/2 cleartext (`h2c`) is supported internally +- HTTP/1 fallback is available on `PORT + 1` +- If no paid search provider is configured, `search` and `serp` still work through the standalone fallback path + +Published image: + +- `ghcr.io/xixu-me/xread:latest` + +For production, pin a digest instead of following `latest`. + +## Deploy It + +The repository includes a complete reverse-proxy deployment example: + +- [Deployment guide](./docs/deploy.md) +- [Container engine guide](./docs/container-engines.md) +- [Compose stack](./deploy/docker-compose.yml) +- [`Caddyfile`](./deploy/Caddyfile) +- [Environment template](./deploy/.env.example) + +The deployment docs use the same image, Compose stack, and environment variables across Docker, Podman, and `nerdctl`/containerd. + +The documented deployment pattern is: + +- `r.your-domain.example` -> `crawl` +- `s.your-domain.example/search` -> `search` +- `s.your-domain.example/?q=...` -> `serp` + +## Configuration + +The standalone build reads configuration from environment variables. + +The deployment template in [`deploy/.env.example`](./deploy/.env.example) includes the shared deployment variables. The full runtime surface is: + +| Variable | Purpose | +| --------------------------------- | --------------------------------------------------------------------------- | +| `PORT` | Main service port. HTTP/1 fallback listens on `PORT + 1`. | +| `PUBLIC_HTTP_PORT` | Public HTTP port exposed by the reverse proxy in the Compose deployment. | +| `PUBLIC_HTTPS_PORT` | Public HTTPS port exposed by the reverse proxy in the Compose deployment. | +| `XREAD_LOOPBACK_HOST` | Host IP used for the backend loopback ports in the Compose deployment. | +| `XREAD_CRAWL_LOOPBACK_PORT` | Host loopback port for the `crawl` backend. | +| `XREAD_SEARCH_LOOPBACK_PORT` | Host loopback port for the `search` backend. | +| `XREAD_SERP_LOOPBACK_PORT` | Host loopback port for the `serp` backend. | +| `STORAGE_ROOT` | Root directory for stored artifacts such as snapshots and generated assets. | +| `LOCAL_DB_ROOT` | Root directory for local metadata, caches, and lightweight persistence. | +| `SERPER_SEARCH_API_KEY` | Enables Serper-backed Google/Bing search. | +| `BRAVE_SEARCH_API_KEY` | Enables Brave Search integration where supported. | +| `CLOUD_FLARE_API_KEY` | Enables Cloudflare-backed capabilities where configured. | +| `LOCAL_PROXY_URLS` | Optional proxy pool for outbound requests. | +| `OVERRIDE_CHROME_EXECUTABLE_PATH` | Optional override for the Chrome executable path. | + +Useful request controls for crawling include: + +- `x-no-cache` +- `x-target-selector` +- `x-wait-for-selector` +- `x-respond-with` +- `x-timeout` + +## Build and Verify + +```bash +npm run lint +npm run security:audit +npm run test:ci +npm run build +``` + +The image build also dry-runs all three standalone servers. + +## Repository Layout + +```text +src/api/ RPC hosts and public HTTP-facing methods +src/stand-alone/ standalone server entrypoints +src/services/ crawling, formatting, search, and runtime services +src/shared/ local shared primitives for storage, rate limits, and config +public/ static assets +deploy/ ready-to-run production deployment templates +docs/ operational documentation +scripts/ build, licensing, and security helper scripts +tests/ regression tests for build, runtime, and automation +``` diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..dbbdd96 --- /dev/null +++ b/README.zh.md @@ -0,0 +1,161 @@ +# Xread + +**_[English](./README.md)_** + +**你的 LLM 值得更好的输入。** + +将杂乱网页转成干净的 Markdown，为 LLM 提供网页搜索能力，并将整套服务自行托管。 + +Xread 是一个面向 LLM 工作流的开源、自托管网页摄取服务，提供三个主要入口： + +- `crawl`：抓取 URL 并转换为可读的 Markdown +- `search`：执行搜索并返回便于直接粘贴进提示词的文本结果 +- `serp`：执行搜索并返回结构化 JSON + +它是公开 [Jina AI Reader](https://jina.ai/reader) 存储库 [jina-ai/reader](https://github.com/jina-ai/reader) 的一个独立分叉版本，移除了原始内部 `thinapps-shared` 依赖，并适配为自托管使用。 + +## 该存储库额外增加了什么 + +- 围绕上游 Reader 存储库完成自托管封装 +- 本地优先的存储与缓存持久化 +- 发布到 GHCR 的 OCI 镜像 +- 面向 Docker、Podman 和 `nerdctl` 的统一镜像、Compose 栈与环境变量部署资产 +- 已接好的 CI、CodeQL、依赖治理与容器发布流程 + +## 快速预览 + +下面这些示例直接使用已发布镜像的 HTTP/1 端口（`8081`）。部署文档会进一步说明 Compose 后端 loopback 端口（`3001`、`3101`、`3201`）以及公网反向代理域名入口。 + +### 抓取网页 + +```bash +curl "http://127.0.0.1:8081/http://example.com" +``` + +### 以文本形式搜索网页 + +```bash +curl "http://127.0.0.1:8081/search?q=example%20domain&num=5&provider=google" +``` + +### 以 JSON 形式搜索网页 + +```bash +curl "http://127.0.0.1:8081/?q=example%20domain&num=5&provider=google" +``` + +## 快速开始 + +### 本地运行 + +```bash +npm ci +npm run build +npm start +``` + +上面会启动 `crawl` 服务。独立入口包括： + +- `build/stand-alone/crawl.js` +- `build/stand-alone/search.js` +- `build/stand-alone/serp.js` + +### 使用容器引擎运行 + +```bash +docker run --rm -p 8081:8081 ghcr.io/xixu-me/xread:latest +``` + +已发布镜像默认启动 `crawl`。任何兼容 OCI 的容器引擎都可以运行它。下面示例使用 Docker 语法；如果要运行其他入口： + +```bash +docker run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/search.js +docker run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/serp.js +``` + +## 为自托管而构建 + +- 镜像内已包含用于页面渲染的 Chrome +- 内部支持 HTTP/2 cleartext（`h2c`） +- 在 `PORT + 1` 上提供 HTTP/1 回退端口 +- 即使没有付费搜索提供方，`search` 和 `serp` 也仍可通过独立回退路径工作 + +已发布镜像： + +- `ghcr.io/xixu-me/xread:latest` + +用于生产环境时，建议固定镜像 digest，而不是持续跟随 `latest`。 + +## 部署 + +存储库内已经提供完整的反向代理部署示例： + +- [部署指南](./docs/deploy.md) +- [容器引擎指南](./docs/container-engines.md) +- [Compose 栈](./deploy/docker-compose.yml) +- [`Caddyfile`](./deploy/Caddyfile) +- [环境模板](./deploy/.env.example) + +部署文档在 Docker、Podman 和 `nerdctl`/containerd 下使用同一镜像、同一 Compose 栈和同一套环境变量。 + +文档中使用的部署拓扑是： + +- `r.your-domain.example` -> `crawl` +- `s.your-domain.example/search` -> `search` +- `s.your-domain.example/?q=...` -> `serp` + +## 配置 + +standalone 构建通过环境变量读取配置。 + +[`deploy/.env.example`](./deploy/.env.example) 中的部署模板包含了共享部署变量。完整的运行时配置面如下： + +| 变量 | 用途 | +| --------------------------------- | --------------------------------------------------- | +| `PORT` | 主服务端口。HTTP/1 回退端口监听在 `PORT + 1`。 | +| `PUBLIC_HTTP_PORT` | Compose 部署中由反向代理暴露的公网 HTTP 端口。 | +| `PUBLIC_HTTPS_PORT` | Compose 部署中由反向代理暴露的公网 HTTPS 端口。 | +| `XREAD_LOOPBACK_HOST` | Compose 部署中后端 loopback 端口所绑定的宿主机 IP。 | +| `XREAD_CRAWL_LOOPBACK_PORT` | `crawl` 后端对应的宿主机 loopback 端口。 | +| `XREAD_SEARCH_LOOPBACK_PORT` | `search` 后端对应的宿主机 loopback 端口。 | +| `XREAD_SERP_LOOPBACK_PORT` | `serp` 后端对应的宿主机 loopback 端口。 | +| `STORAGE_ROOT` | 快照和生成资产等存储内容的根目录。 | +| `LOCAL_DB_ROOT` | 本地元数据、缓存和轻量持久化的根目录。 | +| `SERPER_SEARCH_API_KEY` | 启用基于 Serper 的 Google/Bing 搜索。 | +| `BRAVE_SEARCH_API_KEY` | 在支持的场景下启用 Brave Search 集成。 | +| `CLOUD_FLARE_API_KEY` | 在已配置能力下启用 Cloudflare 相关集成。 | +| `LOCAL_PROXY_URLS` | 可选的出站代理池。 | +| `OVERRIDE_CHROME_EXECUTABLE_PATH` | Chrome 可执行文件路径的可选覆盖值。 | + +抓取相关的常用请求控制头包括： + +- `x-no-cache` +- `x-target-selector` +- `x-wait-for-selector` +- `x-respond-with` +- `x-timeout` + +## 构建与验证 + +```bash +npm run lint +npm run security:audit +npm run test:ci +npm run build +``` + +镜像构建流程还会对三个 standalone 服务做启动演练。 + +## 存储库结构 + +```text +src/api/ RPC 主机与对外 HTTP 方法 +src/stand-alone/ standalone 服务入口 +src/services/ 抓取、格式化、搜索与运行时服务 +src/shared/ 存储、限流与配置等本地共享基础设施 +public/ 静态资源 +deploy/ 可直接使用的生产部署模板 +docs/ 运维文档 +scripts/ 构建、许可证与安全辅助脚本 +tests/ 构建、运行时与自动化回归测试 +``` diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..e2b32eb --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,13 @@ +R_HOST=r.your-domain.example +S_HOST=s.your-domain.example +ACME_EMAIL=ops@your-domain.example +XREAD_IMAGE=ghcr.io/xixu-me/xread:latest +PUBLIC_HTTP_PORT=80 +PUBLIC_HTTPS_PORT=443 +XREAD_LOOPBACK_HOST=127.0.0.1 +XREAD_CRAWL_LOOPBACK_PORT=3001 +XREAD_SEARCH_LOOPBACK_PORT=3101 +XREAD_SERP_LOOPBACK_PORT=3201 +SERPER_SEARCH_API_KEY= +BRAVE_SEARCH_API_KEY= +LOCAL_PROXY_URLS= diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 0000000..90f9d42 --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,17 @@ +{ + email {$ACME_EMAIL} +} + +{$R_HOST} { + encode gzip zstd + reverse_proxy crawl:3001 +} + +{$S_HOST} { + encode gzip zstd + + @text path /search /search/* + reverse_proxy @text search:3101 + + reverse_proxy serp:3201 +} diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..5afec51 --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,75 @@ +services: + crawl: + image: ${XREAD_IMAGE:-ghcr.io/xixu-me/xread:latest} + restart: unless-stopped + environment: + PORT: 3000 + STORAGE_ROOT: /data/storage + LOCAL_DB_ROOT: /data/db + SERPER_SEARCH_API_KEY: ${SERPER_SEARCH_API_KEY:-} + BRAVE_SEARCH_API_KEY: ${BRAVE_SEARCH_API_KEY:-} + LOCAL_PROXY_URLS: ${LOCAL_PROXY_URLS:-} + volumes: + - crawl_storage:/data/storage + - crawl_db:/data/db + ports: + - "${XREAD_LOOPBACK_HOST:-127.0.0.1}:${XREAD_CRAWL_LOOPBACK_PORT:-3001}:3001" + + search: + image: ${XREAD_IMAGE:-ghcr.io/xixu-me/xread:latest} + restart: unless-stopped + command: ["build/stand-alone/search.js"] + environment: + PORT: 3100 + STORAGE_ROOT: /data/storage + LOCAL_DB_ROOT: /data/db + SERPER_SEARCH_API_KEY: ${SERPER_SEARCH_API_KEY:-} + BRAVE_SEARCH_API_KEY: ${BRAVE_SEARCH_API_KEY:-} + LOCAL_PROXY_URLS: ${LOCAL_PROXY_URLS:-} + volumes: + - search_storage:/data/storage + - search_db:/data/db + ports: + - "${XREAD_LOOPBACK_HOST:-127.0.0.1}:${XREAD_SEARCH_LOOPBACK_PORT:-3101}:3101" + + serp: + image: ${XREAD_IMAGE:-ghcr.io/xixu-me/xread:latest} + restart: unless-stopped + command: ["build/stand-alone/serp.js"] + environment: + PORT: 3200 + STORAGE_ROOT: /data/storage + LOCAL_DB_ROOT: /data/db + SERPER_SEARCH_API_KEY: ${SERPER_SEARCH_API_KEY:-} + BRAVE_SEARCH_API_KEY: ${BRAVE_SEARCH_API_KEY:-} + LOCAL_PROXY_URLS: ${LOCAL_PROXY_URLS:-} + volumes: + - serp_storage:/data/storage + - serp_db:/data/db + ports: + - "${XREAD_LOOPBACK_HOST:-127.0.0.1}:${XREAD_SERP_LOOPBACK_PORT:-3201}:3201" + + caddy: + image: caddy:2.10.2-alpine + restart: unless-stopped + ports: + - "${PUBLIC_HTTP_PORT:-80}:80" + - "${PUBLIC_HTTPS_PORT:-443}:443" + environment: + ACME_EMAIL: ${ACME_EMAIL} + R_HOST: ${R_HOST} + S_HOST: ${S_HOST} + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + +volumes: + crawl_storage: + crawl_db: + search_storage: + search_db: + serp_storage: + serp_db: + caddy_data: + caddy_config: diff --git a/docs/container-engines.md b/docs/container-engines.md new file mode 100644 index 0000000..e24f87f --- /dev/null +++ b/docs/container-engines.md @@ -0,0 +1,105 @@ +# Container Engine Guide + +Xread ships as a standard OCI image and a Compose stack. Docker, Podman, and `nerdctl` all use the same image, the same `deploy/docker-compose.yml`, the same environment variables, and the same reverse-proxy layout. + +## Server Runtimes + +| Runtime | Single container | Compose stack | Notes | +| ------------------------- | ---------------- | ------------- | --------------------------------------- | +| Docker Engine | Yes | Yes | Use `docker run` and `docker compose` | +| Podman | Yes | Yes | Use `podman run` and `podman compose` | +| containerd with `nerdctl` | Yes | Yes | Use `nerdctl run` and `nerdctl compose` | + +## Desktop Wrappers + +| Environment | Single container | Compose stack | Notes | +| --------------- | ---------------- | ------------- | -------------------------------------------------------------------------------- | +| Docker Desktop | Yes | Yes | Same commands as Docker Engine | +| OrbStack | Yes | Yes | Docker-compatible CLI | +| Colima | Yes | Yes | Docker-compatible CLI after `docker context` is configured | +| Rancher Desktop | Yes | Yes | Use either Docker-compatible CLI or `nerdctl`, depending on how it is configured | + +> [!NOTE] +> The deployment files in [`deploy`](../deploy) stay named `docker-compose.yml` because that filename is still the broadest common denominator across Compose-compatible tooling. + +## Single-Container Commands + +### Docker-compatible CLIs + +These examples work unchanged with: + +- Docker Engine +- Docker Desktop +- OrbStack +- Colima with Docker context enabled + +```bash +docker run --rm -p 8081:8081 ghcr.io/xixu-me/xread:latest +docker run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/search.js +docker run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/serp.js +``` + +### Podman + +```bash +podman run --rm -p 8081:8081 ghcr.io/xixu-me/xread:latest +podman run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/search.js +podman run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/serp.js +``` + +### nerdctl + +```bash +nerdctl run --rm -p 8081:8081 ghcr.io/xixu-me/xread:latest +nerdctl run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/search.js +nerdctl run --rm -p 8081:8081 --entrypoint node ghcr.io/xixu-me/xread:latest build/stand-alone/serp.js +``` + +## Compose Commands + +The stack in [`deploy/docker-compose.yml`](../deploy/docker-compose.yml) works with Compose-compatible CLIs. All engines use the same file and the same variables from [`deploy/.env.example`](../deploy/.env.example). + +### Docker / Docker Desktop / OrbStack / Colima + +```bash +docker compose pull +docker compose up -d +docker compose ps +``` + +### Podman + +```bash +podman compose pull +podman compose up -d +podman compose ps +``` + +### nerdctl + +```bash +nerdctl compose pull +nerdctl compose up -d +nerdctl compose ps +``` + +## Shared Deployment Knobs + +The deployment model stays the same across engines: + +- `PUBLIC_HTTP_PORT` and `PUBLIC_HTTPS_PORT` control the reverse-proxy entrypoints +- `XREAD_LOOPBACK_HOST` controls whether backend smoke-test ports stay on loopback +- `XREAD_CRAWL_LOOPBACK_PORT`, `XREAD_SEARCH_LOOPBACK_PORT`, and `XREAD_SERP_LOOPBACK_PORT` control the three backend check ports +- `R_HOST`, `S_HOST`, and `ACME_EMAIL` control the public reverse-proxy hostnames and certificate email + +If your host cannot publish `80` and `443`, use the same fix regardless of engine: set `PUBLIC_HTTP_PORT` and `PUBLIC_HTTPS_PORT` to higher ports, or terminate TLS before the stack. + +## Practical Recommendation + +Choose the engine that matches your host standard: + +- Docker Engine +- Podman +- `nerdctl` on containerd + +For local development, Docker Desktop, OrbStack, Colima, and Rancher Desktop all work with the same image and Compose stack. diff --git a/docs/deploy.md b/docs/deploy.md new file mode 100644 index 0000000..9e100cb --- /dev/null +++ b/docs/deploy.md @@ -0,0 +1,243 @@ +# Deploy Xread + +This guide deploys the standalone Xread stack behind Caddy with a Compose-compatible container engine. + +It assumes you want the common public layout: + +- `r.your-domain.example` -> `crawl` +- `s.your-domain.example/search` -> `search` +- `s.your-domain.example/?q=...` -> `serp` + +## What Gets Deployed + +The stack uses four containers: + +- `crawl`: page-to-Markdown service +- `search`: text-oriented web search +- `serp`: structured JSON search +- `caddy`: TLS termination and reverse proxy + +Internally, Caddy talks to the HTTP/1 fallback port on each app container. + +## Runtime Scope + +This deployment layout uses the same image, Compose file, environment variables, and reverse-proxy topology across the common OCI runtimes: + +- Docker Engine +- Docker Desktop +- Podman +- `nerdctl` with containerd +- Docker-compatible desktop wrappers such as OrbStack, Colima, and Rancher Desktop + +If you need engine-specific command names, see [`container-engines.md`](./container-engines.md). + +## Prerequisites + +- A Linux server with an OCI-compatible container engine and a Compose-compatible CLI +- DNS records for your subdomains pointing directly at the server +- Ports `80` and `443` reachable from the public internet + +> [!IMPORTANT] +> Automatic TLS will not work until your public DNS records really resolve to the server's public IP. Verify that first with a public resolver, not only your local machine cache. + +## Files + +Use the templates included in this repository: + +- [`deploy/docker-compose.yml`](../deploy/docker-compose.yml) +- [`deploy/Caddyfile`](../deploy/Caddyfile) +- [`deploy/.env.example`](../deploy/.env.example) + +## 1. Prepare the runtime + +Install one supported runtime on the host, then verify that both the engine CLI and the Compose-compatible CLI are available. The deployment steps after this point stay the same. + +### Docker + +```bash +docker version +docker compose version +``` + +### Podman + +```bash +podman version +podman compose version +``` + +### nerdctl + +```bash +nerdctl version +nerdctl compose version +``` + +Optional but recommended on very small hosts: + +```bash +fallocate -l 1G /swapfile +chmod 600 /swapfile +mkswap /swapfile +swapon /swapfile +grep -q '^/swapfile ' /etc/fstab || echo '/swapfile none swap sw 0 0' >> /etc/fstab +``` + +## 2. Copy the deployment assets + +```bash +mkdir -p /opt/xread +cd /opt/xread +``` + +Copy these files into `/opt/xread`: + +- `docker-compose.yml` +- `Caddyfile` +- `.env` + +Start from the repository template: + +```bash +cp deploy/.env.example /opt/xread/.env +``` + +Choose the Compose-compatible command you plan to use: + +```bash +# Docker / Docker Desktop / OrbStack / Colima +export COMPOSE_CMD="docker compose" + +# Podman +export COMPOSE_CMD="podman compose" + +# nerdctl +export COMPOSE_CMD="nerdctl compose" +``` + +## 3. Configure environment variables + +Edit `/opt/xread/.env`: + +```dotenv +R_HOST=r.your-domain.example +S_HOST=s.your-domain.example +ACME_EMAIL=ops@your-domain.example +XREAD_IMAGE=ghcr.io/xixu-me/xread:latest +PUBLIC_HTTP_PORT=80 +PUBLIC_HTTPS_PORT=443 +XREAD_LOOPBACK_HOST=127.0.0.1 +XREAD_CRAWL_LOOPBACK_PORT=3001 +XREAD_SEARCH_LOOPBACK_PORT=3101 +XREAD_SERP_LOOPBACK_PORT=3201 +SERPER_SEARCH_API_KEY= +BRAVE_SEARCH_API_KEY= +LOCAL_PROXY_URLS= +``` + +Notes: + +- `PUBLIC_HTTP_PORT` and `PUBLIC_HTTPS_PORT` default to `80` and `443` +- If your host or policy does not allow binding `80` and `443`, change those to higher ports such as `8080` and `8443` +- `XREAD_LOOPBACK_HOST` and the three backend loopback ports control the local operator-only smoke-test endpoints +- `SERPER_SEARCH_API_KEY` is optional but recommended for more stable Google/Bing search results +- `BRAVE_SEARCH_API_KEY` is optional +- `LOCAL_PROXY_URLS` is optional and can stay empty +- Other runtime variables exist, but this template keeps the deployment file focused on the common deployment subset + +## 4. Start the stack + +```bash +cd /opt/xread +$COMPOSE_CMD pull +$COMPOSE_CMD up -d +``` + +Check status: + +```bash +$COMPOSE_CMD ps +$COMPOSE_CMD logs --tail=100 +``` + +On small VPS instances, give the stack a little time on first boot. Chrome-based workers can take tens of seconds to warm up. + +## 5. Verify the deployment + +There are three useful checkpoints: + +- direct loopback checks against the three backend containers +- public-domain checks through Caddy +- temporary `--resolve` checks before public DNS is live + +### Local container checks + +These examples use the default loopback values from `deploy/.env.example`: + +```bash +curl http://127.0.0.1:3001/http://example.com +curl "http://127.0.0.1:3101/search?q=example%20domain&num=3&provider=google" +curl "http://127.0.0.1:3201/?q=example%20domain&num=3&provider=google" +``` + +Those loopback ports are published only on `XREAD_LOOPBACK_HOST` for operator smoke tests. They are not exposed publicly unless you change that host binding yourself. + +### Public domain checks + +These examples assume the default public ports `80` and `443`: + +```bash +curl https://r.your-domain.example/http://example.com +curl "https://s.your-domain.example/search?q=example%20domain&num=3&provider=google" +curl "https://s.your-domain.example/?q=example%20domain&num=3&provider=google" +``` + +If DNS is not live yet, you can still smoke-test the reverse proxy path from a client machine: + +```bash +curl --resolve r.your-domain.example:80:YOUR_SERVER_IP -H "Host: r.your-domain.example" http://r.your-domain.example/http://example.com +curl --resolve s.your-domain.example:80:YOUR_SERVER_IP -H "Host: s.your-domain.example" "http://s.your-domain.example/search?q=example%20domain&num=3&provider=google" +``` + +## Operational Notes + +- The app containers store data under named volumes for `storage` and `db` +- Caddy stores certificates and config in named volumes +- The image already contains Chrome and common font packages +- Public traffic terminates at Caddy; app containers stay on the internal Compose network + +## Upgrade + +```bash +cd /opt/xread +$COMPOSE_CMD pull +$COMPOSE_CMD up -d +``` + +For safer upgrades, pin `XREAD_IMAGE` to a digest instead of `latest`. + +## Troubleshooting + +### TLS does not issue + +Usually means one of: + +- DNS is not pointing at the server yet +- Port `80` or `443` is blocked +- Another process is already listening on those ports + +### The host cannot bind 80 or 443 + +Use the same deployment knobs regardless of engine: + +- set `PUBLIC_HTTP_PORT` and `PUBLIC_HTTPS_PORT` to higher ports such as `8080` and `8443` +- terminate TLS on an external load balancer if you want to keep public `443` +- forward `80` and `443` from another service if your host standard requires it + +### Search works poorly without an API key + +That is expected. The standalone fallback path is useful for bootstrapping and low-volume use, but provider-backed search is more stable. + +### Browser rendering fails on small hosts + +Make sure swap is enabled and the host is not memory-starved. The image includes Chrome, so low-memory VPS instances can benefit from at least `1G` swap. diff --git a/eslint.config.cjs b/eslint.config.cjs index 3d5977e..48bb5f8 100644 --- a/eslint.config.cjs +++ b/eslint.config.cjs @@ -1,103 +1,103 @@ -const tsPlugin = require('@typescript-eslint/eslint-plugin'); -const tsParser = require('@typescript-eslint/parser'); +const tsPlugin = require("@typescript-eslint/eslint-plugin"); +const tsParser = require("@typescript-eslint/parser"); const sharedRules = { - 'no-empty': 'off', - 'no-useless-escape': 'warn', - 'no-prototype-builtins': 'off', - 'no-case-declarations': 'off', - 'no-control-regex': 'off', - 'no-async-promise-executor': 'off', - 'no-constant-condition': 'off', - 'no-extra-boolean-cast': 'off', - 'no-extra-semi': 'off', - 'no-unsafe-optional-chaining': 'off', - 'no-undef': 'off', - 'no-var': 'off', - 'prefer-const': 'off', - 'prefer-rest-params': 'off', - 'require-yield': 'off', + "no-empty": "off", + "no-useless-escape": "warn", + "no-prototype-builtins": "off", + "no-case-declarations": "off", + "no-control-regex": "off", + "no-async-promise-executor": "off", + "no-constant-condition": "off", + "no-extra-boolean-cast": "off", + "no-extra-semi": "off", + "no-unsafe-optional-chaining": "off", + "no-undef": "off", + "no-var": "off", + "prefer-const": "off", + "prefer-rest-params": "off", + "require-yield": "off", }; module.exports = [ - { - ignores: [ - 'build/**', - 'node_modules/**', - 'licensed/**', - '.codex-cache/**', - 'coverage/**', - ], + { + ignores: [ + "build/**", + "node_modules/**", + "licensed/**", + ".codex-cache/**", + "coverage/**", + ], + }, + { + files: ["**/*.ts"], + languageOptions: { + parser: tsParser, + ecmaVersion: "latest", + sourceType: "module", }, - { - files: ['**/*.ts'], - languageOptions: { - parser: tsParser, - ecmaVersion: 'latest', - sourceType: 'module', - }, - plugins: { - '@typescript-eslint': tsPlugin, - }, - rules: { - ...tsPlugin.configs['flat/recommended'][1].rules, - ...tsPlugin.configs['flat/recommended'][2].rules, - ...sharedRules, - '@typescript-eslint/no-explicit-any': 'off', - '@typescript-eslint/no-inferrable-types': 'off', - '@typescript-eslint/no-non-null-assertion': 'off', - '@typescript-eslint/no-require-imports': 'off', - '@typescript-eslint/no-var-requires': 'off', - '@typescript-eslint/ban-ts-comment': 'off', - '@typescript-eslint/no-extra-semi': 'off', - '@typescript-eslint/no-this-alias': 'off', - '@typescript-eslint/ban-types': 'off', - '@typescript-eslint/no-unsafe-declaration-merging': 'off', - '@typescript-eslint/no-wrapper-object-types': 'off', - '@typescript-eslint/no-unused-vars': [ - 'warn', - { - argsIgnorePattern: '^_', - varsIgnorePattern: '^_', - caughtErrorsIgnorePattern: '^_', - }, - ], - }, + plugins: { + "@typescript-eslint": tsPlugin, }, - { - files: ['**/*.js'], - languageOptions: { - ecmaVersion: 'latest', - sourceType: 'module', - }, - rules: { - ...sharedRules, - 'no-unused-vars': [ - 'warn', - { - argsIgnorePattern: '^_', - varsIgnorePattern: '^_', - caughtErrorsIgnorePattern: '^_', - }, - ], + rules: { + ...tsPlugin.configs["flat/recommended"][1].rules, + ...tsPlugin.configs["flat/recommended"][2].rules, + ...sharedRules, + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-inferrable-types": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/no-require-imports": "off", + "@typescript-eslint/no-var-requires": "off", + "@typescript-eslint/ban-ts-comment": "off", + "@typescript-eslint/no-extra-semi": "off", + "@typescript-eslint/no-this-alias": "off", + "@typescript-eslint/ban-types": "off", + "@typescript-eslint/no-unsafe-declaration-merging": "off", + "@typescript-eslint/no-wrapper-object-types": "off", + "@typescript-eslint/no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + caughtErrorsIgnorePattern: "^_", }, + ], }, - { - files: ['**/*.cjs'], - languageOptions: { - ecmaVersion: 'latest', - sourceType: 'script', - }, - rules: { - ...sharedRules, - 'no-unused-vars': [ - 'warn', - { - argsIgnorePattern: '^_', - varsIgnorePattern: '^_', - caughtErrorsIgnorePattern: '^_', - }, - ], - }, + }, + { + files: ["**/*.js"], + languageOptions: { + ecmaVersion: "latest", + sourceType: "module", }, + rules: { + ...sharedRules, + "no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + caughtErrorsIgnorePattern: "^_", + }, + ], + }, + }, + { + files: ["**/*.cjs"], + languageOptions: { + ecmaVersion: "latest", + sourceType: "script", + }, + rules: { + ...sharedRules, + "no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + varsIgnorePattern: "^_", + caughtErrorsIgnorePattern: "^_", + }, + ], + }, + }, ]; diff --git a/integrity-check.cjs b/integrity-check.cjs index f9e6b1e..f0477f9 100644 --- a/integrity-check.cjs +++ b/integrity-check.cjs @@ -1,10 +1,12 @@ #!/usr/bin/env node -const fs = require('fs'); -const path = require('path'); +const fs = require("fs"); +const path = require("path"); -const file = path.resolve(__dirname, 'licensed/GeoLite2-City.mmdb'); +const file = path.resolve(__dirname, "licensed/GeoLite2-City.mmdb"); if (!fs.existsSync(file)) { - console.error(`Integrity check warning: ${file} does not exist. GeoIP features will be disabled until the asset is prepared.`); + console.error( + `Integrity check warning: ${file} does not exist. GeoIP features will be disabled until the asset is prepared.`, + ); } diff --git a/package-lock.json b/package-lock.json index 5f6e1c6..3f20a01 100644 --- a/package-lock.json +++ b/package-lock.json @@ -58,6 +58,7 @@ "@typescript-eslint/parser": "^8.57.2", "eslint": "^10.1.0", "pino-pretty": "^13.1.3", + "prettier": "^3.8.1", "typescript": "^5.9.3" }, "engines": { @@ -5635,6 +5636,22 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.1.tgz", + "integrity": "sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", diff --git a/package.json b/package.json index 6be1a42..0111c2e 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,8 @@ "name": "xread", "scripts": { "lint": "eslint --ext .js,.ts .", + "format": "prettier --write .", + "format:check": "prettier --check .", "security:audit": "node ./scripts/security-audit-report.cjs", "test:ci": "node --test tests/open-source-build.test.cjs tests/integrity-check.test.cjs tests/prepare-licensed-assets.test.cjs tests/github-automation.test.cjs tests/container-runtime-regressions.test.cjs tests/security-governance.test.cjs", "prepare:licensed-assets": "node ./scripts/prepare-licensed-assets.cjs", @@ -70,6 +72,7 @@ "@typescript-eslint/parser": "^8.57.2", "eslint": "^10.1.0", "pino-pretty": "^13.1.3", + "prettier": "^3.8.1", "typescript": "^5.9.3" }, "overrides": { diff --git a/public/favicon.ico b/public/favicon.ico deleted file mode 100644 index 1f9ee24..0000000 Binary files a/public/favicon.ico and /dev/null differ diff --git a/scripts/prepare-licensed-assets.cjs b/scripts/prepare-licensed-assets.cjs index f934447..f64feda 100644 --- a/scripts/prepare-licensed-assets.cjs +++ b/scripts/prepare-licensed-assets.cjs @@ -1,19 +1,21 @@ #!/usr/bin/env node -const fs = require('node:fs'); -const path = require('node:path'); +const fs = require("node:fs"); +const path = require("node:path"); const DEFAULT_ASSETS = [ { - path: 'licensed/GeoLite2-City.mmdb', - url: 'https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb', + path: "licensed/GeoLite2-City.mmdb", + url: "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb", }, ]; async function downloadAsset(asset, destination) { const response = await fetch(asset.url); if (!response.ok) { - throw new Error(`Failed to download ${asset.url}: ${response.status} ${response.statusText}`); + throw new Error( + `Failed to download ${asset.url}: ${response.status} ${response.statusText}`, + ); } const arrayBuffer = await response.arrayBuffer(); @@ -22,7 +24,7 @@ async function downloadAsset(asset, destination) { } async function ensureLicensedAssets({ - rootDir = __dirname ? path.resolve(__dirname, '..') : process.cwd(), + rootDir = __dirname ? path.resolve(__dirname, "..") : process.cwd(), assets = DEFAULT_ASSETS, downloadAsset: downloadImpl = downloadAsset, } = {}) { @@ -39,7 +41,7 @@ async function ensureLicensedAssets({ if (require.main === module) { ensureLicensedAssets() .then(() => { - process.stdout.write('Licensed assets are ready.\n'); + process.stdout.write("Licensed assets are ready.\n"); }) .catch((error) => { process.stderr.write(`${error.stack || error}\n`); diff --git a/scripts/security-audit-report.cjs b/scripts/security-audit-report.cjs index 5593e6a..dcf3380 100644 --- a/scripts/security-audit-report.cjs +++ b/scripts/security-audit-report.cjs @@ -1,12 +1,16 @@ -const fs = require('node:fs'); -const path = require('node:path'); -const { spawnSync } = require('node:child_process'); +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); -const projectRoot = path.resolve(__dirname, '..'); -const baselinePath = path.join(projectRoot, 'security', 'npm-audit-baseline.json'); -const reportDirectory = path.join(projectRoot, 'security-reports'); -const summaryJsonPath = path.join(reportDirectory, 'npm-audit-summary.json'); -const summaryMarkdownPath = path.join(reportDirectory, 'npm-audit-summary.md'); +const projectRoot = path.resolve(__dirname, ".."); +const baselinePath = path.join( + projectRoot, + "security", + "npm-audit-baseline.json", +); +const reportDirectory = path.join(projectRoot, "security-reports"); +const summaryJsonPath = path.join(reportDirectory, "npm-audit-summary.json"); +const summaryMarkdownPath = path.join(reportDirectory, "npm-audit-summary.md"); const severityRank = { info: 0, @@ -17,29 +21,32 @@ const severityRank = { }; function npmCommand() { - return process.platform === 'win32' ? 'npm.cmd' : 'npm'; + return process.platform === "win32" ? "npm.cmd" : "npm"; } function readBaseline() { - const raw = fs.readFileSync(baselinePath, 'utf8'); + const raw = fs.readFileSync(baselinePath, "utf8"); const parsed = JSON.parse(raw); return Array.isArray(parsed.entries) ? parsed.entries : []; } function runAudit(extraArgs) { - const result = spawnSync(npmCommand(), ['audit', '--json', ...extraArgs], { + const result = spawnSync(npmCommand(), ["audit", "--json", ...extraArgs], { cwd: projectRoot, - encoding: 'utf8', + encoding: "utf8", env: process.env, }); if (![0, 1].includes(result.status ?? 0)) { - process.stderr.write(result.stderr || result.stdout || 'npm audit failed.\n'); + process.stderr.write( + result.stderr || result.stdout || "npm audit failed.\n", + ); process.exit(result.status ?? 1); } - const combinedOutput = `${result.stdout || ''}\n${result.stderr || ''}`.trim(); - const jsonStart = combinedOutput.indexOf('{'); + const combinedOutput = + `${result.stdout || ""}\n${result.stderr || ""}`.trim(); + const jsonStart = combinedOutput.indexOf("{"); if (jsonStart === -1) { if ((result.status ?? 0) === 0) { return { @@ -58,7 +65,7 @@ function runAudit(extraArgs) { }; } - throw new Error('npm audit returned no JSON payload.'); + throw new Error("npm audit returned no JSON payload."); } return JSON.parse(combinedOutput.slice(jsonStart)); @@ -67,7 +74,7 @@ function runAudit(extraArgs) { function toFindings(report, scope) { return Object.entries(report.vulnerabilities || {}).map(([pkg, entry]) => { const advisories = (entry.via || []) - .filter((item) => item && typeof item === 'object' && item.title) + .filter((item) => item && typeof item === "object" && item.title) .map((item) => ({ source: item.source, title: item.title, @@ -88,10 +95,11 @@ function toFindings(report, scope) { } function matchBaseline(finding, baselineEntries) { - return baselineEntries.find((entry) => - entry.package === finding.package && - entry.scope === finding.scope && - entry.severity === finding.severity + return baselineEntries.find( + (entry) => + entry.package === finding.package && + entry.scope === finding.scope && + entry.severity === finding.severity, ); } @@ -100,54 +108,60 @@ function formatFinding(finding, baselineEntry) { const notes = []; if (finding.advisories.length) { - notes.push(`advisories: ${finding.advisories.map((item) => item.title).join(' | ')}`); + notes.push( + `advisories: ${finding.advisories.map((item) => item.title).join(" | ")}`, + ); } if (finding.effects.length) { - notes.push(`effects: ${finding.effects.join(', ')}`); + notes.push(`effects: ${finding.effects.join(", ")}`); } if (finding.nodes.length) { - notes.push(`nodes: ${finding.nodes.join(', ')}`); + notes.push(`nodes: ${finding.nodes.join(", ")}`); } if (baselineEntry) { - notes.push(`baseline: ${baselineEntry.reason} (review by ${baselineEntry.reviewBy})`); + notes.push( + `baseline: ${baselineEntry.reason} (review by ${baselineEntry.reviewBy})`, + ); } - return `- ${headline}\n ${notes.join('\n ')}`; + return `- ${headline}\n ${notes.join("\n ")}`; } function writeReports(payload) { fs.mkdirSync(reportDirectory, { recursive: true }); - fs.writeFileSync(summaryJsonPath, JSON.stringify(payload, null, 2) + '\n'); + fs.writeFileSync(summaryJsonPath, JSON.stringify(payload, null, 2) + "\n"); const sections = [ - '# npm audit summary', - '', + "# npm audit summary", + "", `- generatedAt: ${payload.generatedAt}`, `- blockingFindings: ${payload.blockingFindings.length}`, `- baselinedDevelopmentFindings: ${payload.baselinedDevelopmentFindings.length}`, `- productionVulnerabilities: ${JSON.stringify(payload.production.metadata.vulnerabilities)}`, `- fullAuditVulnerabilities: ${JSON.stringify(payload.full.metadata.vulnerabilities)}`, - '', - '## Blocking findings', + "", + "## Blocking findings", ...(payload.blockingFindings.length ? payload.blockingFindings.map((item) => formatFinding(item, null)) - : ['- none']), - '', - '## Baselined development findings', + : ["- none"]), + "", + "## Baselined development findings", ...(payload.baselinedDevelopmentFindings.length - ? payload.baselinedDevelopmentFindings.map((item) => formatFinding(item.finding, item.baseline)) - : ['- none']), + ? payload.baselinedDevelopmentFindings.map((item) => + formatFinding(item.finding, item.baseline), + ) + : ["- none"]), ]; - fs.writeFileSync(summaryMarkdownPath, sections.join('\n') + '\n'); + fs.writeFileSync(summaryMarkdownPath, sections.join("\n") + "\n"); } const baselineEntries = readBaseline(); -const productionAudit = runAudit(['--omit=dev']); +const productionAudit = runAudit(["--omit=dev"]); const fullAudit = runAudit([]); -const productionFindings = toFindings(productionAudit, 'production'); -const fullFindings = toFindings(fullAudit, 'development'); +const productionFindings = toFindings(productionAudit, "production"); +const fullFindings = toFindings(fullAudit, "development"); const baselinedDevelopmentFindings = []; const blockingFindings = []; @@ -159,7 +173,11 @@ for (const finding of productionFindings) { } for (const finding of fullFindings) { - if (productionFindings.some((prodFinding) => prodFinding.package === finding.package)) { + if ( + productionFindings.some( + (prodFinding) => prodFinding.package === finding.package, + ) + ) { continue; } @@ -191,8 +209,10 @@ const payload = { writeReports(payload); if (blockingFindings.length) { - process.stderr.write(`Security audit failed with ${blockingFindings.length} blocking finding(s).\n`); + process.stderr.write( + `Security audit failed with ${blockingFindings.length} blocking finding(s).\n`, + ); process.exit(1); } -process.stdout.write('Security audit policy passed.\n'); +process.stdout.write("Security audit policy passed.\n"); diff --git a/scripts/transpile.cjs b/scripts/transpile.cjs index dbf8603..4e686c1 100644 --- a/scripts/transpile.cjs +++ b/scripts/transpile.cjs @@ -1,17 +1,17 @@ #!/usr/bin/env node -const fs = require('node:fs'); -const path = require('node:path'); -const childProcess = require('node:child_process'); +const fs = require("node:fs"); +const path = require("node:path"); +const childProcess = require("node:child_process"); -const projectRoot = path.resolve(__dirname, '..'); -const tsconfigPath = path.join(projectRoot, 'tsconfig.json'); -const bootstrapPrefix = path.join(projectRoot, '.codex-cache', 'ts-compiler'); +const projectRoot = path.resolve(__dirname, ".."); +const tsconfigPath = path.join(projectRoot, "tsconfig.json"); +const bootstrapPrefix = path.join(projectRoot, ".codex-cache", "ts-compiler"); function resolveTypeScript() { const candidatePaths = [ - path.join(projectRoot, 'node_modules', 'typescript'), - path.join(bootstrapPrefix, 'node_modules', 'typescript'), + path.join(projectRoot, "node_modules", "typescript"), + path.join(bootstrapPrefix, "node_modules", "typescript"), ]; for (const candidate of candidatePaths) { @@ -22,20 +22,30 @@ function resolveTypeScript() { ensureDir(bootstrapPrefix); const install = childProcess.spawnSync( - 'npm', - ['install', '--prefix', bootstrapPrefix, '--no-save', '--ignore-scripts', '--no-package-lock', 'typescript@5.5.4'], + "npm", + [ + "install", + "--prefix", + bootstrapPrefix, + "--no-save", + "--ignore-scripts", + "--no-package-lock", + "typescript@5.5.4", + ], { cwd: projectRoot, - stdio: 'inherit', + stdio: "inherit", shell: true, }, ); if (install.status !== 0) { - throw new Error(`Unable to bootstrap TypeScript compiler (exit ${install.status ?? 'unknown'})`); + throw new Error( + `Unable to bootstrap TypeScript compiler (exit ${install.status ?? "unknown"})`, + ); } - return require(path.join(bootstrapPrefix, 'node_modules', 'typescript')); + return require(path.join(bootstrapPrefix, "node_modules", "typescript")); } const ts = resolveTypeScript(); @@ -54,7 +64,9 @@ function cleanDir(dirPath) { function loadConfig() { const rawConfig = ts.readConfigFile(tsconfigPath, ts.sys.readFile); if (rawConfig.error) { - throw new Error(ts.flattenDiagnosticMessageText(rawConfig.error.messageText, '\n')); + throw new Error( + ts.flattenDiagnosticMessageText(rawConfig.error.messageText, "\n"), + ); } const parsed = ts.parseJsonConfigFileContent( @@ -72,24 +84,24 @@ function loadConfig() { }, fileNames: parsed.fileNames.filter((fileName) => { const normalized = path.resolve(fileName); - if (!normalized.startsWith(path.join(projectRoot, 'src'))) { + if (!normalized.startsWith(path.join(projectRoot, "src"))) { return false; } - return !normalized.endsWith('.d.ts'); + return !normalized.endsWith(".d.ts"); }), - outDir: path.resolve(projectRoot, parsed.options.outDir || 'build'), + outDir: path.resolve(projectRoot, parsed.options.outDir || "build"), }; } function outputPathFor(fileName, outDir) { - const relative = path.relative(path.join(projectRoot, 'src'), fileName); + const relative = path.relative(path.join(projectRoot, "src"), fileName); const ext = path.extname(relative); const base = relative.slice(0, relative.length - ext.length); return path.join(outDir, `${base}.js`); } function transpileFile(fileName, compilerOptions, outDir) { - const sourceText = fs.readFileSync(fileName, 'utf8'); + const sourceText = fs.readFileSync(fileName, "utf8"); const transpiled = ts.transpileModule(sourceText, { compilerOptions, fileName, @@ -98,15 +110,18 @@ function transpileFile(fileName, compilerOptions, outDir) { const outputFile = outputPathFor(fileName, outDir); ensureDir(path.dirname(outputFile)); - fs.writeFileSync(outputFile, transpiled.outputText, 'utf8'); + fs.writeFileSync(outputFile, transpiled.outputText, "utf8"); if (transpiled.sourceMapText) { - fs.writeFileSync(`${outputFile}.map`, transpiled.sourceMapText, 'utf8'); + fs.writeFileSync(`${outputFile}.map`, transpiled.sourceMapText, "utf8"); } if (transpiled.diagnostics?.length) { for (const diagnostic of transpiled.diagnostics) { - const message = ts.flattenDiagnosticMessageText(diagnostic.messageText, '\n'); + const message = ts.flattenDiagnosticMessageText( + diagnostic.messageText, + "\n", + ); process.stderr.write(`[transpile warning] ${fileName}: ${message}\n`); } } @@ -120,7 +135,9 @@ function main() { transpileFile(fileName, compilerOptions, outDir); } - process.stdout.write(`Transpiled ${fileNames.length} source files to ${outDir}\n`); + process.stdout.write( + `Transpiled ${fileNames.length} source files to ${outDir}\n`, + ); } try { diff --git a/src/api/crawler.ts b/src/api/crawler.ts index 00fbab6..f1d3393 100644 --- a/src/api/crawler.ts +++ b/src/api/crawler.ts @@ -1,1379 +1,1826 @@ -import { singleton } from 'tsyringe'; -import { pathToFileURL } from 'url'; -import { randomUUID } from 'crypto'; -import _ from 'lodash'; +import { singleton } from "tsyringe"; +import { pathToFileURL } from "url"; +import { randomUUID } from "crypto"; +import _ from "lodash"; import { - assignTransferProtocolMeta, RPCHost, RPCReflection, - AssertionFailureError, ParamValidationError, - RawString, - ApplicationError, - DataStreamBrokenError, - assignMeta, -} from 'civkit/civ-rpc'; -import { marshalErrorLike } from 'civkit/lang'; -import { Defer } from 'civkit/defer'; -import { retryWith } from 'civkit/decorators'; -import { FancyFile } from 'civkit/fancy-file'; + assignTransferProtocolMeta, + RPCHost, + RPCReflection, + AssertionFailureError, + ParamValidationError, + RawString, + ApplicationError, + DataStreamBrokenError, + assignMeta, +} from "civkit/civ-rpc"; +import { marshalErrorLike } from "civkit/lang"; +import { Defer } from "civkit/defer"; +import { retryWith } from "civkit/decorators"; +import { FancyFile } from "civkit/fancy-file"; -import { CONTENT_FORMAT, CrawlerOptions, CrawlerOptionsHeaderOnly, ENGINE_TYPE, RESPOND_TIMING } from '../dto/crawler-options'; - -import { Crawled } from '../db/crawled'; -import { DomainBlockade } from '../db/domain-blockade'; -import { OutputServerEventStream } from '../lib/transform-server-event-stream'; - -import { PageSnapshot, PuppeteerControl, ScrappingOptions } from '../services/puppeteer'; -import { JSDomControl } from '../services/jsdom'; -import { FormattedPage, md5Hasher, SnapshotFormatter } from '../services/snapshot-formatter'; -import { CurlControl } from '../services/curl'; -import { LmControl } from '../services/lm'; -import { tryDecodeURIComponent } from '../utils/misc'; -import { CFBrowserRendering } from '../services/cf-browser-rendering'; -import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; - -import { GlobalLogger } from '../services/logger'; -import { RateLimitControl, RateLimitDesc } from '../shared/services/rate-limit'; -import { AsyncLocalContext } from '../services/async-context'; -import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry'; import { - BudgetExceededError, InsufficientBalanceError, - SecurityCompromiseError, ServiceBadApproachError, ServiceBadAttemptError, - ServiceNodeResourceDrainError -} from '../services/errors'; + CONTENT_FORMAT, + CrawlerOptions, + CrawlerOptionsHeaderOnly, + ENGINE_TYPE, + RESPOND_TIMING, +} from "../dto/crawler-options"; -import { countGPTToken as estimateToken } from '../shared/utils/openai'; -import { ProxyProviderService } from '../shared/services/proxy-provider'; -import { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket'; -import { AuthDTO } from '../dto/auth'; -import { RobotsTxtService } from '../services/robots-text'; -import { TempFileManager } from '../services/temp-file'; -import { MiscService } from '../services/misc'; -import { HTTPServiceError } from 'civkit/http'; -import { GeoIPService } from '../services/geoip'; +import { Crawled } from "../db/crawled"; +import { DomainBlockade } from "../db/domain-blockade"; +import { OutputServerEventStream } from "../lib/transform-server-event-stream"; + +import { + PageSnapshot, + PuppeteerControl, + ScrappingOptions, +} from "../services/puppeteer"; +import { JSDomControl } from "../services/jsdom"; +import { + FormattedPage, + md5Hasher, + SnapshotFormatter, +} from "../services/snapshot-formatter"; +import { CurlControl } from "../services/curl"; +import { LmControl } from "../services/lm"; +import { tryDecodeURIComponent } from "../utils/misc"; +import { CFBrowserRendering } from "../services/cf-browser-rendering"; +import { STANDALONE_BOOT_TIMEOUT_MS } from "../services/boot-timeouts"; + +import { GlobalLogger } from "../services/logger"; +import { RateLimitControl, RateLimitDesc } from "../shared/services/rate-limit"; +import { AsyncLocalContext } from "../services/async-context"; +import { Context, Ctx, Method, Param, RPCReflect } from "../services/registry"; +import { + BudgetExceededError, + InsufficientBalanceError, + SecurityCompromiseError, + ServiceBadApproachError, + ServiceBadAttemptError, + ServiceNodeResourceDrainError, +} from "../services/errors"; + +import { countGPTToken as estimateToken } from "../shared/utils/openai"; +import { ProxyProviderService } from "../shared/services/proxy-provider"; +import { FirebaseStorageBucketControl } from "../shared/services/firebase-storage-bucket"; +import { AuthDTO } from "../dto/auth"; +import { RobotsTxtService } from "../services/robots-text"; +import { TempFileManager } from "../services/temp-file"; +import { MiscService } from "../services/misc"; +import { HTTPServiceError } from "civkit/http"; +import { GeoIPService } from "../services/geoip"; export interface ExtraScrappingOptions extends ScrappingOptions { - withIframe?: boolean | 'quoted'; - withShadowDom?: boolean; - targetSelector?: string | string[]; - removeSelector?: string | string[]; - keepImgDataUrl?: boolean; - engine?: string; - allocProxy?: string; - private?: boolean; - countryHint?: string; + withIframe?: boolean | "quoted"; + withShadowDom?: boolean; + targetSelector?: string | string[]; + removeSelector?: string | string[]; + keepImgDataUrl?: boolean; + engine?: string; + allocProxy?: string; + private?: boolean; + countryHint?: string; } const indexProto = { - toString: function (): string { - return _(this) - .toPairs() - .map(([k, v]) => k ? `[${_.upperFirst(_.lowerCase(k))}] ${v}` : '') - .value() - .join('\n') + '\n'; - } + toString: function (): string { + return ( + _(this) + .toPairs() + .map(([k, v]) => (k ? `[${_.upperFirst(_.lowerCase(k))}] ${v}` : "")) + .value() + .join("\n") + "\n" + ); + }, }; +function derivePublicOrigins(ctx?: Context) { + const homepage = "https://github.com/xixu-me/xread"; + const fallbackOrigin = "http://127.0.0.1:8081"; + const currentOrigin = ctx?.origin || fallbackOrigin; + + try { + const currentUrl = new URL(currentOrigin); + let crawlOrigin = currentUrl.origin; + let searchOrigin = currentUrl.origin; + + if (ctx?.hostname.startsWith("r.")) { + const derivedSearchUrl = new URL(currentUrl.origin); + derivedSearchUrl.hostname = `s.${ctx.hostname.slice(2)}`; + searchOrigin = derivedSearchUrl.origin; + } else if (ctx?.hostname.startsWith("s.")) { + const derivedCrawlUrl = new URL(currentUrl.origin); + derivedCrawlUrl.hostname = `r.${ctx.hostname.slice(2)}`; + crawlOrigin = derivedCrawlUrl.origin; + } + + return { crawlOrigin, searchOrigin, homepage }; + } catch { + return { + crawlOrigin: fallbackOrigin, + searchOrigin: fallbackOrigin, + homepage, + }; + } +} + @singleton() export class CrawlerHost extends RPCHost { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - cacheRetentionMs = 1000 * 3600 * 24 * 7; - cacheValidMs = 1000 * 3600; - urlValidMs = 1000 * 3600 * 4; - abuseBlockMs = 1000 * 3600; - domainProfileRetentionMs = 1000 * 3600 * 24 * 30; + cacheRetentionMs = 1000 * 3600 * 24 * 7; + cacheValidMs = 1000 * 3600; + urlValidMs = 1000 * 3600 * 4; + abuseBlockMs = 1000 * 3600; + domainProfileRetentionMs = 1000 * 3600 * 24 * 30; - batchedCaches: Crawled[] = []; + batchedCaches: Crawled[] = []; - constructor( - protected globalLogger: GlobalLogger, - protected puppeteerControl: PuppeteerControl, - protected curlControl: CurlControl, - protected cfBrowserRendering: CFBrowserRendering, - protected proxyProvider: ProxyProviderService, - protected lmControl: LmControl, - protected jsdomControl: JSDomControl, - protected snapshotFormatter: SnapshotFormatter, - protected firebaseObjectStorage: FirebaseStorageBucketControl, - protected rateLimitControl: RateLimitControl, - protected threadLocal: AsyncLocalContext, - protected robotsTxtService: RobotsTxtService, - protected tempFileManager: TempFileManager, - protected geoIpService: GeoIPService, - protected miscService: MiscService, - ) { - super(...arguments); + constructor( + protected globalLogger: GlobalLogger, + protected puppeteerControl: PuppeteerControl, + protected curlControl: CurlControl, + protected cfBrowserRendering: CFBrowserRendering, + protected proxyProvider: ProxyProviderService, + protected lmControl: LmControl, + protected jsdomControl: JSDomControl, + protected snapshotFormatter: SnapshotFormatter, + protected firebaseObjectStorage: FirebaseStorageBucketControl, + protected rateLimitControl: RateLimitControl, + protected threadLocal: AsyncLocalContext, + protected robotsTxtService: RobotsTxtService, + protected tempFileManager: TempFileManager, + protected geoIpService: GeoIPService, + protected miscService: MiscService, + ) { + super(...arguments); - puppeteerControl.on('crawled', async (snapshot: PageSnapshot, options: ExtraScrappingOptions & { url: URL; }) => { - if (!snapshot.title?.trim() && !snapshot.pdfs?.length) { - return; - } - if (options.cookies?.length || options.private) { - // Potential privacy issue, dont cache if cookies are used - return; - } - if (options.injectFrameScripts?.length || options.injectPageScripts?.length || options.viewport) { - // Potentially mangeled content, dont cache if scripts are injected - return; - } - if (snapshot.isIntermediate) { - return; - } - if (!snapshot.lastMutationIdle) { - // Never reached mutationIdle, presumably too short timeout - return; - } - if (options.locale) { - Reflect.set(snapshot, 'locale', options.locale); - } - - const analyzed = await this.jsdomControl.analyzeHTMLTextLite(snapshot.html); - if (analyzed.tokens < 200) { - // Does not contain enough content - if (snapshot.status !== 200) { - return; - } - if (snapshot.html.includes('captcha') || snapshot.html.includes('cf-turnstile')) { - return; - } - } - - await this.setToCache(options.url, snapshot); - }); - - puppeteerControl.on('abuse', async (abuseEvent: { url: URL; reason: string, sn: number; }) => { - this.logger.warn(`Abuse detected on ${abuseEvent.url}, blocking ${abuseEvent.url.hostname}`, { reason: abuseEvent.reason, sn: abuseEvent.sn }); - - await DomainBlockade.save(DomainBlockade.from({ - domain: abuseEvent.url.hostname.toLowerCase(), - triggerReason: `${abuseEvent.reason}`, - triggerUrl: abuseEvent.url.toString(), - createdAt: new Date(), - expireAt: new Date(Date.now() + this.abuseBlockMs), - })).catch((err) => { - this.logger.warn(`Failed to save domain blockade for ${abuseEvent.url.hostname}`, { err: marshalErrorLike(err) }); - }); - - }); - - setInterval(() => { - const thisBatch = this.batchedCaches; - this.batchedCaches = []; - if (!thisBatch.length) { - return; - } - const batch = Crawled.DB.batch(); - - for (const x of thisBatch) { - batch.set(Crawled.COLLECTION.doc(x._id), x.degradeForFireStore(), { merge: true }); - } - - batch.commit() - .then(() => { - this.logger.debug(`Saved ${thisBatch.length} caches by batch`); - }) - .catch((err) => { - this.logger.warn(`Failed to save cache in batch`, { err }); - }); - }, 1000 * 10 + Math.round(1000 * Math.random())).unref(); - } - - override async init() { - await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); - - if (this.puppeteerControl.effectiveUA) { - this.curlControl.impersonateChrome(this.puppeteerControl.effectiveUA); + puppeteerControl.on( + "crawled", + async ( + snapshot: PageSnapshot, + options: ExtraScrappingOptions & { url: URL }, + ) => { + if (!snapshot.title?.trim() && !snapshot.pdfs?.length) { + return; + } + if (options.cookies?.length || options.private) { + // Potential privacy issue, dont cache if cookies are used + return; + } + if ( + options.injectFrameScripts?.length || + options.injectPageScripts?.length || + options.viewport + ) { + // Potentially mangeled content, dont cache if scripts are injected + return; + } + if (snapshot.isIntermediate) { + return; + } + if (!snapshot.lastMutationIdle) { + // Never reached mutationIdle, presumably too short timeout + return; + } + if (options.locale) { + Reflect.set(snapshot, "locale", options.locale); } - this.emit('ready'); - } - - override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { - return super.dependencyReady(timeoutMilliseconds); - } - - async getIndex(auth?: AuthDTO) { - const indexObject: Record = Object.create(indexProto); - Object.assign(indexObject, { - usage1: 'https://r.example.com/YOUR_URL', - usage2: 'https://s.example.com/YOUR_SEARCH_QUERY', - homepage: 'https://example.com/xread', - }); - - await auth?.solveUID(); - if (auth && auth.user) { - indexObject[''] = undefined; - indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`; - indexObject.balanceLeft = auth.user.wallet.total_balance; - } - - return indexObject; - } - - @Method({ - name: 'getIndex', - description: 'Index of the service', - proto: { - http: { - action: 'get', - path: '/', - } - }, - tags: ['misc', 'crawl'], - returnType: [String, Object], - }) - async getIndexCtrl(@Ctx() ctx: Context, @Param({ required: false }) auth?: AuthDTO) { - const indexObject = await this.getIndex(auth); - - if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) { - return indexObject; - } - - return assignTransferProtocolMeta(`${indexObject}`, - { contentType: 'text/plain; charset=utf-8', envelope: null } + const analyzed = await this.jsdomControl.analyzeHTMLTextLite( + snapshot.html, ); - } - - - @Method({ - name: 'crawlByPostingToIndex', - description: 'Crawl any url into markdown', - proto: { - http: { - action: 'POST', - path: '/', - } - }, - tags: ['crawl'], - returnType: [String, OutputServerEventStream], - }) - @Method({ - description: 'Crawl any url into markdown', - proto: { - http: { - action: ['GET', 'POST'], - path: '::url', - } - }, - tags: ['crawl'], - returnType: [String, OutputServerEventStream, RawString], - }) - async crawl( - @RPCReflect() rpcReflect: RPCReflection, - @Ctx() ctx: Context, - auth: AuthDTO, - crawlerOptionsHeaderOnly: CrawlerOptionsHeaderOnly, - crawlerOptionsParamsAllowed: CrawlerOptions, - ) { - const uid = await auth.solveUID(); - let chargeAmount = 0; - const crawlerOptions = ctx.method === 'GET' ? crawlerOptionsHeaderOnly : crawlerOptionsParamsAllowed; - const tierPolicy = await this.saasAssertTierPolicy(crawlerOptions, auth); - - // Use koa ctx.URL, a standard URL object to avoid node.js framework prop naming confusion - const targetUrl = await this.getTargetUrl(tryDecodeURIComponent(`${ctx.URL.pathname}${ctx.URL.search}`), crawlerOptions); - if (!targetUrl) { - return await this.getIndex(auth); - } - - // Prevent circular crawling - this.puppeteerControl.circuitBreakerHosts.add( - ctx.hostname.toLowerCase() - ); - - if (uid) { - const user = await auth.assertUser(); - if (!(user.wallet.total_balance > 0)) { - throw new InsufficientBalanceError(`Account balance not enough to run this query, please recharge.`); - } - - const rateLimitPolicy = auth.getRateLimits('CRAWL') || [ - parseInt(user.metadata?.speed_level) >= 2 ? - RateLimitDesc.from({ - occurrence: 5000, - periodSeconds: 60 - }) : - RateLimitDesc.from({ - occurrence: 500, - periodSeconds: 60 - }) - ]; - - const apiRoll = await this.rateLimitControl.simpleRPCUidBasedLimit( - rpcReflect, uid, ['CRAWL'], - ...rateLimitPolicy - ); - - rpcReflect.finally(() => { - if (crawlerOptions.tokenBudget && chargeAmount > crawlerOptions.tokenBudget) { - return; - } - if (chargeAmount) { - auth.reportUsage(chargeAmount, `xread-crawl`).catch((err) => { - this.logger.warn(`Unable to report usage for ${uid}`, { err: marshalErrorLike(err) }); - }); - apiRoll.chargeAmount = chargeAmount; - } - }); - } else if (ctx.ip) { - const apiRoll = await this.rateLimitControl.simpleRpcIPBasedLimit(rpcReflect, ctx.ip, ['CRAWL'], - [ - // 20 requests per minute - new Date(Date.now() - 60 * 1000), 20 - ] - ); - - rpcReflect.finally(() => { - if (crawlerOptions.tokenBudget && chargeAmount > crawlerOptions.tokenBudget) { - return; - } - apiRoll.chargeAmount = chargeAmount; - }); - } - - if (!uid) { - // Enforce no proxy is allocated for anonymous users due to abuse. - crawlerOptions.proxy = 'none'; - const blockade = (await DomainBlockade.fromFirestoreQuery( - DomainBlockade.COLLECTION - .where('domain', '==', targetUrl.hostname.toLowerCase()) - .where('expireAt', '>=', new Date()) - .limit(1) - ))[0]; - if (blockade) { - throw new SecurityCompromiseError(`Domain ${targetUrl.hostname} blocked until ${blockade.expireAt || 'Eternally'} due to previous abuse found on ${blockade.triggerUrl || 'site'}: ${blockade.triggerReason}`); - } - } - const crawlOpts = await this.configure(crawlerOptions); - this.logger.info(`Accepting request from ${uid || ctx.ip}`, { opts: crawlerOptions }); - if (crawlerOptions.robotsTxt) { - await this.robotsTxtService.assertAccessAllowed(targetUrl, crawlerOptions.robotsTxt); - } - if (rpcReflect.signal.aborted) { + if (analyzed.tokens < 200) { + // Does not contain enough content + if (snapshot.status !== 200) { return; - } - if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) { - const sseStream = new OutputServerEventStream(); - rpcReflect.return(sseStream); - - try { - for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) { - if (!scrapped) { - continue; - } - if (rpcReflect.signal.aborted) { - break; - } - - const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts); - chargeAmount = this.assignChargeAmount(formatted, tierPolicy); - sseStream.write({ - event: 'data', - data: formatted, - }); - if (chargeAmount && scrapped.pdfs?.length) { - break; - } - } - } catch (err: any) { - this.logger.error(`Failed to crawl ${targetUrl}`, { err: marshalErrorLike(err) }); - sseStream.write({ - event: 'error', - data: marshalErrorLike(err), - }); - } - - sseStream.end(); - - return sseStream; + } + if ( + snapshot.html.includes("captcha") || + snapshot.html.includes("cf-turnstile") + ) { + return; + } } - let lastScrapped; - if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) { - try { - for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) { - lastScrapped = scrapped; - if (rpcReflect.signal.aborted) { - break; - } - if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) { - continue; - } - if (!scrapped.title) { - continue; - } + await this.setToCache(options.url, snapshot); + }, + ); - const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts); - chargeAmount = this.assignChargeAmount(formatted, tierPolicy); + puppeteerControl.on( + "abuse", + async (abuseEvent: { url: URL; reason: string; sn: number }) => { + this.logger.warn( + `Abuse detected on ${abuseEvent.url}, blocking ${abuseEvent.url.hostname}`, + { reason: abuseEvent.reason, sn: abuseEvent.sn }, + ); - if (scrapped?.pdfs?.length && !chargeAmount) { - continue; - } + await DomainBlockade.save( + DomainBlockade.from({ + domain: abuseEvent.url.hostname.toLowerCase(), + triggerReason: `${abuseEvent.reason}`, + triggerUrl: abuseEvent.url.toString(), + createdAt: new Date(), + expireAt: new Date(Date.now() + this.abuseBlockMs), + }), + ).catch((err) => { + this.logger.warn( + `Failed to save domain blockade for ${abuseEvent.url.hostname}`, + { err: marshalErrorLike(err) }, + ); + }); + }, + ); - return formatted; - } - } catch (err) { - if (!lastScrapped) { - throw err; - } - } + setInterval( + () => { + const thisBatch = this.batchedCaches; + this.batchedCaches = []; + if (!thisBatch.length) { + return; + } + const batch = Crawled.DB.batch(); - if (!lastScrapped) { - if (crawlOpts.targetSelector) { - throw new AssertionFailureError(`No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(', ') : crawlOpts.targetSelector}`); - } - throw new AssertionFailureError(`No content available for URL ${targetUrl}`); - } - - const formatted = await this.formatSnapshot(crawlerOptions, lastScrapped, targetUrl, this.urlValidMs, crawlOpts); - chargeAmount = this.assignChargeAmount(formatted, tierPolicy); - - return formatted; + for (const x of thisBatch) { + batch.set(Crawled.COLLECTION.doc(x._id), x.degradeForFireStore(), { + merge: true, + }); } - if (crawlerOptions.isRequestingCompoundContentFormat()) { - throw new ParamValidationError({ - path: 'respondWith', - message: `You are requesting compound content format, please explicitly accept 'text/event-stream' or 'application/json' in header.` + batch + .commit() + .then(() => { + this.logger.debug(`Saved ${thisBatch.length} caches by batch`); + }) + .catch((err) => { + this.logger.warn(`Failed to save cache in batch`, { err }); + }); + }, + 1000 * 10 + Math.round(1000 * Math.random()), + ).unref(); + } + + override async init() { + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + + if (this.puppeteerControl.effectiveUA) { + this.curlControl.impersonateChrome(this.puppeteerControl.effectiveUA); + } + + this.emit("ready"); + } + + override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { + return super.dependencyReady(timeoutMilliseconds); + } + + async getIndex(ctx?: Context, auth?: AuthDTO) { + const { crawlOrigin, homepage } = derivePublicOrigins(ctx); + const indexObject: Record = + Object.create(indexProto); + Object.assign(indexObject, { + crawl: `${crawlOrigin}/YOUR_URL`, + repository: homepage, + }); + + await auth?.solveUID(); + if (auth && auth.user) { + indexObject[""] = undefined; + indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`; + indexObject.balanceLeft = auth.user.wallet.total_balance; + } + + return indexObject; + } + + @Method({ + name: "getIndex", + description: "Index of the service", + proto: { + http: { + action: "get", + path: "/", + }, + }, + tags: ["misc", "crawl"], + returnType: [String, Object], + }) + async getIndexCtrl( + @Ctx() ctx: Context, + @Param({ required: false }) auth?: AuthDTO, + ) { + const indexObject = await this.getIndex(ctx, auth); + + if ( + !ctx.accepts("text/plain") && + (ctx.accepts("text/json") || ctx.accepts("application/json")) + ) { + return indexObject; + } + + return assignTransferProtocolMeta(`${indexObject}`, { + contentType: "text/plain; charset=utf-8", + envelope: null, + }); + } + + @Method({ + name: "crawlByPostingToIndex", + description: "Crawl any url into markdown", + proto: { + http: { + action: "POST", + path: "/", + }, + }, + tags: ["crawl"], + returnType: [String, OutputServerEventStream], + }) + @Method({ + description: "Crawl any url into markdown", + proto: { + http: { + action: ["GET", "POST"], + path: "::url", + }, + }, + tags: ["crawl"], + returnType: [String, OutputServerEventStream, RawString], + }) + async crawl( + @RPCReflect() rpcReflect: RPCReflection, + @Ctx() ctx: Context, + auth: AuthDTO, + crawlerOptionsHeaderOnly: CrawlerOptionsHeaderOnly, + crawlerOptionsParamsAllowed: CrawlerOptions, + ) { + const uid = await auth.solveUID(); + let chargeAmount = 0; + const crawlerOptions = + ctx.method === "GET" + ? crawlerOptionsHeaderOnly + : crawlerOptionsParamsAllowed; + const tierPolicy = await this.saasAssertTierPolicy(crawlerOptions, auth); + + // Use koa ctx.URL, a standard URL object to avoid node.js framework prop naming confusion + const targetUrl = await this.getTargetUrl( + tryDecodeURIComponent(`${ctx.URL.pathname}${ctx.URL.search}`), + crawlerOptions, + ); + if (!targetUrl) { + return await this.getIndex(ctx, auth); + } + + // Prevent circular crawling + this.puppeteerControl.circuitBreakerHosts.add(ctx.hostname.toLowerCase()); + + if (uid) { + const user = await auth.assertUser(); + if (!(user.wallet.total_balance > 0)) { + throw new InsufficientBalanceError( + `Account balance not enough to run this query, please recharge.`, + ); + } + + const rateLimitPolicy = auth.getRateLimits("CRAWL") || [ + parseInt(user.metadata?.speed_level) >= 2 + ? RateLimitDesc.from({ + occurrence: 5000, + periodSeconds: 60, + }) + : RateLimitDesc.from({ + occurrence: 500, + periodSeconds: 60, + }), + ]; + + const apiRoll = await this.rateLimitControl.simpleRPCUidBasedLimit( + rpcReflect, + uid, + ["CRAWL"], + ...rateLimitPolicy, + ); + + rpcReflect.finally(() => { + if ( + crawlerOptions.tokenBudget && + chargeAmount > crawlerOptions.tokenBudget + ) { + return; + } + if (chargeAmount) { + auth.reportUsage(chargeAmount, `xread-crawl`).catch((err) => { + this.logger.warn(`Unable to report usage for ${uid}`, { + err: marshalErrorLike(err), }); + }); + apiRoll.chargeAmount = chargeAmount; } + }); + } else if (ctx.ip) { + const apiRoll = await this.rateLimitControl.simpleRpcIPBasedLimit( + rpcReflect, + ctx.ip, + ["CRAWL"], + [ + // 20 requests per minute + new Date(Date.now() - 60 * 1000), + 20, + ], + ); - try { - for await (const scrapped of this.iterSnapshots(targetUrl, crawlOpts, crawlerOptions)) { - lastScrapped = scrapped; - if (rpcReflect.signal.aborted) { - break; - } - if (!scrapped || !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped)) { - continue; - } - if (!scrapped.title) { - continue; - } - - const formatted = await this.formatSnapshot(crawlerOptions, scrapped, targetUrl, this.urlValidMs, crawlOpts); - chargeAmount = this.assignChargeAmount(formatted, tierPolicy); - - if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) { - return assignTransferProtocolMeta(`${formatted.textRepresentation}`, - { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } } - ); - } - if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) { - return assignTransferProtocolMeta(`${formatted.textRepresentation}`, - { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } } - ); - } - - return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null }); - } - } catch (err) { - if (!lastScrapped) { - throw err; - } + rpcReflect.finally(() => { + if ( + crawlerOptions.tokenBudget && + chargeAmount > crawlerOptions.tokenBudget + ) { + return; } + apiRoll.chargeAmount = chargeAmount; + }); + } + if (!uid) { + // Enforce no proxy is allocated for anonymous users due to abuse. + crawlerOptions.proxy = "none"; + const blockade = ( + await DomainBlockade.fromFirestoreQuery( + DomainBlockade.COLLECTION.where( + "domain", + "==", + targetUrl.hostname.toLowerCase(), + ) + .where("expireAt", ">=", new Date()) + .limit(1), + ) + )[0]; + if (blockade) { + throw new SecurityCompromiseError( + `Domain ${targetUrl.hostname} blocked until ${blockade.expireAt || "Eternally"} due to previous abuse found on ${blockade.triggerUrl || "site"}: ${blockade.triggerReason}`, + ); + } + } + const crawlOpts = await this.configure(crawlerOptions); + this.logger.info(`Accepting request from ${uid || ctx.ip}`, { + opts: crawlerOptions, + }); + if (crawlerOptions.robotsTxt) { + await this.robotsTxtService.assertAccessAllowed( + targetUrl, + crawlerOptions.robotsTxt, + ); + } + if (rpcReflect.signal.aborted) { + return; + } + if (!ctx.accepts("text/plain") && ctx.accepts("text/event-stream")) { + const sseStream = new OutputServerEventStream(); + rpcReflect.return(sseStream); + + try { + for await (const scrapped of this.iterSnapshots( + targetUrl, + crawlOpts, + crawlerOptions, + )) { + if (!scrapped) { + continue; + } + if (rpcReflect.signal.aborted) { + break; + } + + const formatted = await this.formatSnapshot( + crawlerOptions, + scrapped, + targetUrl, + this.urlValidMs, + crawlOpts, + ); + chargeAmount = this.assignChargeAmount(formatted, tierPolicy); + sseStream.write({ + event: "data", + data: formatted, + }); + if (chargeAmount && scrapped.pdfs?.length) { + break; + } + } + } catch (err: any) { + this.logger.error(`Failed to crawl ${targetUrl}`, { + err: marshalErrorLike(err), + }); + sseStream.write({ + event: "error", + data: marshalErrorLike(err), + }); + } + + sseStream.end(); + + return sseStream; + } + + let lastScrapped; + if ( + !ctx.accepts("text/plain") && + (ctx.accepts("text/json") || ctx.accepts("application/json")) + ) { + try { + for await (const scrapped of this.iterSnapshots( + targetUrl, + crawlOpts, + crawlerOptions, + )) { + lastScrapped = scrapped; + if (rpcReflect.signal.aborted) { + break; + } + if ( + !scrapped || + !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped) + ) { + continue; + } + if (!scrapped.title) { + continue; + } + + const formatted = await this.formatSnapshot( + crawlerOptions, + scrapped, + targetUrl, + this.urlValidMs, + crawlOpts, + ); + chargeAmount = this.assignChargeAmount(formatted, tierPolicy); + + if (scrapped?.pdfs?.length && !chargeAmount) { + continue; + } + + return formatted; + } + } catch (err) { if (!lastScrapped) { - if (crawlOpts.targetSelector) { - throw new AssertionFailureError(`No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(', ') : crawlOpts.targetSelector}`); - } - throw new AssertionFailureError(`No content available for URL ${targetUrl}`); + throw err; } - const formatted = await this.formatSnapshot(crawlerOptions, lastScrapped, targetUrl, this.urlValidMs, crawlOpts); + } + + if (!lastScrapped) { + if (crawlOpts.targetSelector) { + throw new AssertionFailureError( + `No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(", ") : crawlOpts.targetSelector}`, + ); + } + throw new AssertionFailureError( + `No content available for URL ${targetUrl}`, + ); + } + + const formatted = await this.formatSnapshot( + crawlerOptions, + lastScrapped, + targetUrl, + this.urlValidMs, + crawlOpts, + ); + chargeAmount = this.assignChargeAmount(formatted, tierPolicy); + + return formatted; + } + + if (crawlerOptions.isRequestingCompoundContentFormat()) { + throw new ParamValidationError({ + path: "respondWith", + message: `You are requesting compound content format, please explicitly accept 'text/event-stream' or 'application/json' in header.`, + }); + } + + try { + for await (const scrapped of this.iterSnapshots( + targetUrl, + crawlOpts, + crawlerOptions, + )) { + lastScrapped = scrapped; + if (rpcReflect.signal.aborted) { + break; + } + if ( + !scrapped || + !crawlerOptions.isSnapshotAcceptableForEarlyResponse(scrapped) + ) { + continue; + } + if (!scrapped.title) { + continue; + } + + const formatted = await this.formatSnapshot( + crawlerOptions, + scrapped, + targetUrl, + this.urlValidMs, + crawlOpts, + ); chargeAmount = this.assignChargeAmount(formatted, tierPolicy); - if (crawlerOptions.respondWith === 'screenshot' && Reflect.get(formatted, 'screenshotUrl')) { + if ( + crawlerOptions.respondWith === "screenshot" && + Reflect.get(formatted, "screenshotUrl") + ) { + return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { + code: 302, + envelope: null, + headers: { Location: Reflect.get(formatted, "screenshotUrl") }, + }); + } + if ( + crawlerOptions.respondWith === "pageshot" && + Reflect.get(formatted, "pageshotUrl") + ) { + return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { + code: 302, + envelope: null, + headers: { Location: Reflect.get(formatted, "pageshotUrl") }, + }); + } - return assignTransferProtocolMeta(`${formatted.textRepresentation}`, - { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'screenshotUrl') } } + return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { + contentType: "text/plain; charset=utf-8", + envelope: null, + }); + } + } catch (err) { + if (!lastScrapped) { + throw err; + } + } + + if (!lastScrapped) { + if (crawlOpts.targetSelector) { + throw new AssertionFailureError( + `No content available for URL ${targetUrl} with target selector ${Array.isArray(crawlOpts.targetSelector) ? crawlOpts.targetSelector.join(", ") : crawlOpts.targetSelector}`, + ); + } + throw new AssertionFailureError( + `No content available for URL ${targetUrl}`, + ); + } + const formatted = await this.formatSnapshot( + crawlerOptions, + lastScrapped, + targetUrl, + this.urlValidMs, + crawlOpts, + ); + chargeAmount = this.assignChargeAmount(formatted, tierPolicy); + + if ( + crawlerOptions.respondWith === "screenshot" && + Reflect.get(formatted, "screenshotUrl") + ) { + return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { + code: 302, + envelope: null, + headers: { Location: Reflect.get(formatted, "screenshotUrl") }, + }); + } + if ( + crawlerOptions.respondWith === "pageshot" && + Reflect.get(formatted, "pageshotUrl") + ) { + return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { + code: 302, + envelope: null, + headers: { Location: Reflect.get(formatted, "pageshotUrl") }, + }); + } + + return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { + contentType: "text/plain; charset=utf-8", + envelope: null, + }); + } + + async getTargetUrl(originPath: string, crawlerOptions: CrawlerOptions) { + let url: string = ""; + + const targetUrlFromGet = originPath.slice(1); + if (crawlerOptions.pdf) { + const pdfFile = crawlerOptions.pdf; + const identifier = + pdfFile instanceof FancyFile ? await pdfFile.sha256Sum : randomUUID(); + url = `blob://pdf/${identifier}`; + crawlerOptions.url ??= url; + } else if (targetUrlFromGet) { + url = targetUrlFromGet.trim(); + } else if (crawlerOptions.url) { + url = crawlerOptions.url.trim(); + } + + if (!url) { + throw new ParamValidationError({ + message: "No URL provided", + path: "url", + }); + } + + const { url: safeURL, ips } = + await this.miscService.assertNormalizedUrl(url); + if ( + this.puppeteerControl.circuitBreakerHosts.has( + safeURL.hostname.toLowerCase(), + ) + ) { + throw new SecurityCompromiseError({ + message: `Circular hostname: ${safeURL.protocol}`, + path: "url", + }); + } + crawlerOptions._hintIps = ips; + + return safeURL; + } + + getUrlDigest(urlToCrawl: URL) { + const normalizedURL = new URL(urlToCrawl); + if (!normalizedURL.hash.startsWith("#/")) { + normalizedURL.hash = ""; + } + const normalizedUrl = normalizedURL.toString().toLowerCase(); + const digest = md5Hasher.hash(normalizedUrl.toString()); + + return digest; + } + + async *queryCache(urlToCrawl: URL, cacheTolerance: number) { + const digest = this.getUrlDigest(urlToCrawl); + + const cache = ( + await Crawled.fromFirestoreQuery( + Crawled.COLLECTION.where("urlPathDigest", "==", digest) + .orderBy("createdAt", "desc") + .limit(1), + ).catch((err) => { + this.logger.warn(`Failed to query cache, unknown issue`, { err }); + // https://github.com/grpc/grpc-node/issues/2647 + // https://github.com/googleapis/nodejs-firestore/issues/1023 + // https://github.com/googleapis/nodejs-firestore/issues/1023 + + return undefined; + }) + )?.[0]; + + yield cache; + + if (!cache) { + return; + } + + const age = Date.now() - cache.createdAt.valueOf(); + const stale = cache.createdAt.valueOf() < Date.now() - cacheTolerance; + this.logger.info( + `${stale ? "Stale cache exists" : "Cache hit"} for ${urlToCrawl}, normalized digest: ${digest}, ${age}ms old, tolerance ${cacheTolerance}ms`, + { + url: urlToCrawl, + digest, + age, + stale, + cacheTolerance, + }, + ); + + let snapshot: PageSnapshot | undefined; + let screenshotUrl: string | undefined; + let pageshotUrl: string | undefined; + const preparations = [ + this.firebaseObjectStorage + .downloadFile(`snapshots/${cache._id}`) + .then((r) => { + snapshot = JSON.parse(r.toString("utf-8")); + }), + cache.screenshotAvailable + ? this.firebaseObjectStorage + .signDownloadUrl( + `screenshots/${cache._id}`, + Date.now() + this.urlValidMs, + ) + .then((r) => { + screenshotUrl = r; + }) + : Promise.resolve(undefined), + cache.pageshotAvailable + ? this.firebaseObjectStorage + .signDownloadUrl( + `pageshots/${cache._id}`, + Date.now() + this.urlValidMs, + ) + .then((r) => { + pageshotUrl = r; + }) + : Promise.resolve(undefined), + ]; + try { + await Promise.all(preparations); + } catch (_err) { + // Swallow cache errors. + return undefined; + } + + yield { + isFresh: !stale, + ...cache, + snapshot: { + ...snapshot, + screenshot: undefined, + pageshot: undefined, + screenshotUrl, + pageshotUrl, + } as PageSnapshot & { screenshotUrl?: string; pageshotUrl?: string }, + }; + } + + async setToCache(urlToCrawl: URL, snapshot: PageSnapshot) { + const digest = this.getUrlDigest(urlToCrawl); + + this.logger.info(`Caching snapshot of ${urlToCrawl}...`, { + url: urlToCrawl, + digest, + title: snapshot?.title, + href: snapshot?.href, + }); + const nowDate = new Date(); + + const cache = Crawled.from({ + _id: randomUUID(), + url: urlToCrawl.toString(), + createdAt: nowDate, + expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs), + htmlSignificantlyModifiedByJs: snapshot.htmlSignificantlyModifiedByJs, + urlPathDigest: digest, + }); + + const savingOfSnapshot = this.firebaseObjectStorage + .saveFile( + `snapshots/${cache._id}`, + Buffer.from( + JSON.stringify({ + ...snapshot, + screenshot: undefined, + pageshot: undefined, + }), + "utf-8", + ), + { + metadata: { + contentType: "application/json", + }, + }, + ) + .then((r) => { + cache.snapshotAvailable = true; + return r; + }); + + if (snapshot.screenshot) { + await this.firebaseObjectStorage.saveFile( + `screenshots/${cache._id}`, + snapshot.screenshot, + { + metadata: { + contentType: "image/png", + }, + }, + ); + cache.screenshotAvailable = true; + } + if (snapshot.pageshot) { + await this.firebaseObjectStorage.saveFile( + `pageshots/${cache._id}`, + snapshot.pageshot, + { + metadata: { + contentType: "image/png", + }, + }, + ); + cache.pageshotAvailable = true; + } + await savingOfSnapshot; + this.batchedCaches.push(cache); + // const r = await Crawled.save(cache.degradeForFireStore()).catch((err) => { + // this.logger.error(`Failed to save cache for ${urlToCrawl}`, { err: marshalErrorLike(err) }); + + // return undefined; + // }); + + return cache; + } + + async *iterSnapshots( + urlToCrawl: URL, + crawlOpts?: ExtraScrappingOptions, + crawlerOpts?: CrawlerOptions, + ) { + // if (crawlerOpts?.respondWith.includes(CONTENT_FORMAT.VLM)) { + // const finalBrowserSnapshot = await this.getFinalSnapshot(urlToCrawl, { + // ...crawlOpts, engine: ENGINE_TYPE.BROWSER + // }, crawlerOpts); + + // yield* this.lmControl.geminiFromBrowserSnapshot(finalBrowserSnapshot); + + // return; + // } + + if (crawlerOpts?.respondWith.includes(CONTENT_FORMAT.XREAD_LM)) { + const finalAutoSnapshot = await this.getFinalSnapshot( + urlToCrawl, + { + ...crawlOpts, + engine: crawlOpts?.engine || ENGINE_TYPE.AUTO, + }, + CrawlerOptions.from({ + ...crawlerOpts, + respondWith: "html", + }), + ); + + if (!finalAutoSnapshot?.html) { + throw new AssertionFailureError( + `Unexpected non HTML content for XreadLM: ${urlToCrawl}`, + ); + } + + if (crawlerOpts?.instruction || crawlerOpts?.jsonSchema) { + const jsonSchema = crawlerOpts.jsonSchema + ? JSON.stringify(crawlerOpts.jsonSchema, undefined, 2) + : undefined; + yield* this.lmControl.xreadLMFromSnapshot( + crawlerOpts.instruction, + jsonSchema, + finalAutoSnapshot, + ); + + return; + } + + try { + yield* this.lmControl.xreadLMMarkdownFromSnapshot(finalAutoSnapshot); + } catch (err) { + if (err instanceof HTTPServiceError && err.status === 429) { + throw new ServiceNodeResourceDrainError( + `Xread LM is at capacity, please try again later.`, + ); + } + throw err; + } + + return; + } + + yield* this.cachedScrap(urlToCrawl, crawlOpts, crawlerOpts); + } + + async *cachedScrap( + urlToCrawl: URL, + crawlOpts?: ExtraScrappingOptions, + crawlerOpts?: CrawlerOptions, + ) { + if (crawlerOpts?.html) { + const snapshot = { + href: urlToCrawl.toString(), + html: crawlerOpts.html, + title: "", + text: "", + } as PageSnapshot; + yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts); + + return; + } + + if (crawlerOpts?.pdf) { + const pdfFile = + crawlerOpts.pdf instanceof FancyFile + ? crawlerOpts.pdf + : this.tempFileManager.cacheBuffer( + Buffer.from(crawlerOpts.pdf, "base64"), ); - } - if (crawlerOptions.respondWith === 'pageshot' && Reflect.get(formatted, 'pageshotUrl')) { + const pdfLocalPath = pathToFileURL(await pdfFile.filePath); + const snapshot = { + href: urlToCrawl.toString(), + html: ``, + title: "", + text: "", + pdfs: [pdfLocalPath.href], + } as PageSnapshot; - return assignTransferProtocolMeta(`${formatted.textRepresentation}`, - { code: 302, envelope: null, headers: { Location: Reflect.get(formatted, 'pageshotUrl') } } + yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts); + + return; + } + + if ( + crawlOpts?.engine === ENGINE_TYPE.CURL || + // deprecated name + crawlOpts?.engine === "direct" + ) { + let sideLoaded; + try { + sideLoaded = + crawlOpts?.allocProxy && !crawlOpts?.proxyUrl + ? await this.sideLoadWithAllocatedProxy(urlToCrawl, crawlOpts) + : await this.curlControl.sideLoad(urlToCrawl, crawlOpts); + } catch (err) { + if (err instanceof ServiceBadAttemptError) { + throw new AssertionFailureError(err.message); + } + throw err; + } + if (!sideLoaded?.file) { + throw new AssertionFailureError( + `Remote server did not return a body: ${urlToCrawl}`, + ); + } + const draftSnapshot = await this.snapshotFormatter.createSnapshotFromFile( + urlToCrawl, + sideLoaded.file, + sideLoaded.contentType, + sideLoaded.fileName, + ); + draftSnapshot.status = sideLoaded.status; + draftSnapshot.statusText = sideLoaded.statusText; + yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts); + return; + } + if (crawlOpts?.engine === ENGINE_TYPE.CF_BROWSER_RENDERING) { + const html = await this.cfBrowserRendering.fetchContent(urlToCrawl.href); + const snapshot = { + href: urlToCrawl.toString(), + html, + title: "", + text: "", + } as PageSnapshot; + yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts); + return; + } + + const cacheTolerance = crawlerOpts?.cacheTolerance ?? this.cacheValidMs; + const cacheIt = this.queryCache(urlToCrawl, cacheTolerance); + + let cache = (await cacheIt.next()).value; + if (cache?.htmlSignificantlyModifiedByJs === false) { + if (crawlerOpts && crawlerOpts.timeout === undefined) { + crawlerOpts.respondTiming ??= RESPOND_TIMING.HTML; + } + } + + if (!crawlerOpts || crawlerOpts.isCacheQueryApplicable()) { + cache = (await cacheIt.next()).value; + } + cacheIt.return(undefined); + + if ( + cache?.isFresh && + (!crawlOpts?.favorScreenshot || + (crawlOpts?.favorScreenshot && + cache.screenshotAvailable && + cache.pageshotAvailable)) && + _.get(cache.snapshot, "locale") === crawlOpts?.locale + ) { + if (cache.snapshot) { + cache.snapshot.isFromCache = true; + } + yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts); + + return; + } + + if ( + crawlOpts?.engine !== ENGINE_TYPE.BROWSER && + !this.knownUrlThatSideLoadingWouldCrashTheBrowser(urlToCrawl) + ) { + const sideLoadSnapshotPermitted = + crawlerOpts?.browserIsNotRequired() && + [RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes( + crawlerOpts.presumedRespondTiming, + ); + try { + const altOpts = { ...crawlOpts }; + let sideLoaded = + crawlOpts?.allocProxy && !crawlOpts?.proxyUrl + ? await this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts) + : await this.curlControl + .sideLoad(urlToCrawl, altOpts) + .catch((err) => { + this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { + err: marshalErrorLike(err), + href: urlToCrawl.href, + }); + + if ( + err instanceof ApplicationError && + !(err instanceof ServiceBadAttemptError) + ) { + return Promise.reject(err); + } + + return this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts); + }); + if (!sideLoaded.file) { + throw new ServiceBadAttemptError( + `Remote server did not return a body: ${urlToCrawl}`, + ); + } + const draftSnapshot = await this.snapshotFormatter + .createSnapshotFromFile( + urlToCrawl, + sideLoaded.file, + sideLoaded.contentType, + sideLoaded.fileName, + ) + .catch((err) => { + if (err instanceof ApplicationError) { + return Promise.reject(new ServiceBadAttemptError(err.message)); + } + return Promise.reject(err); + }); + draftSnapshot.status = sideLoaded.status; + draftSnapshot.statusText = sideLoaded.statusText; + if ( + sideLoaded.status == 200 && + !sideLoaded.contentType.startsWith("text/html") + ) { + yield draftSnapshot; + return; + } + + let analyzed = await this.jsdomControl.analyzeHTMLTextLite( + draftSnapshot.html, + ); + draftSnapshot.title ??= analyzed.title; + draftSnapshot.isIntermediate = true; + if (sideLoadSnapshotPermitted) { + yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts); + } + let fallbackProxyIsUsed = false; + if ( + (!crawlOpts?.allocProxy || crawlOpts.allocProxy !== "none") && + !crawlOpts?.proxyUrl && + (analyzed.tokens < 42 || sideLoaded.status !== 200) + ) { + const proxyLoaded = await this.sideLoadWithAllocatedProxy( + urlToCrawl, + altOpts, + ); + if (!proxyLoaded.file) { + throw new ServiceBadAttemptError( + `Remote server did not return a body: ${urlToCrawl}`, ); - } - - return assignTransferProtocolMeta(`${formatted.textRepresentation}`, { contentType: 'text/plain; charset=utf-8', envelope: null }); - - } - - async getTargetUrl(originPath: string, crawlerOptions: CrawlerOptions) { - let url: string = ''; - - const targetUrlFromGet = originPath.slice(1); - if (crawlerOptions.pdf) { - const pdfFile = crawlerOptions.pdf; - const identifier = pdfFile instanceof FancyFile ? (await pdfFile.sha256Sum) : randomUUID(); - url = `blob://pdf/${identifier}`; - crawlerOptions.url ??= url; - } else if (targetUrlFromGet) { - url = targetUrlFromGet.trim(); - } else if (crawlerOptions.url) { - url = crawlerOptions.url.trim(); - } - - if (!url) { - throw new ParamValidationError({ - message: 'No URL provided', - path: 'url' + } + const proxySnapshot = await this.snapshotFormatter + .createSnapshotFromFile( + urlToCrawl, + proxyLoaded.file, + proxyLoaded.contentType, + proxyLoaded.fileName, + ) + .catch((err) => { + if (err instanceof ApplicationError) { + return Promise.reject(new ServiceBadAttemptError(err.message)); + } + return Promise.reject(err); }); - } - - const { url: safeURL, ips } = await this.miscService.assertNormalizedUrl(url); - if (this.puppeteerControl.circuitBreakerHosts.has(safeURL.hostname.toLowerCase())) { - throw new SecurityCompromiseError({ - message: `Circular hostname: ${safeURL.protocol}`, - path: 'url' - }); - } - crawlerOptions._hintIps = ips; - - return safeURL; - } - - getUrlDigest(urlToCrawl: URL) { - const normalizedURL = new URL(urlToCrawl); - if (!normalizedURL.hash.startsWith('#/')) { - normalizedURL.hash = ''; - } - const normalizedUrl = normalizedURL.toString().toLowerCase(); - const digest = md5Hasher.hash(normalizedUrl.toString()); - - return digest; - } - - async *queryCache(urlToCrawl: URL, cacheTolerance: number) { - const digest = this.getUrlDigest(urlToCrawl); - - const cache = ( - await - (Crawled.fromFirestoreQuery( - Crawled.COLLECTION.where('urlPathDigest', '==', digest).orderBy('createdAt', 'desc').limit(1) - ).catch((err) => { - this.logger.warn(`Failed to query cache, unknown issue`, { err }); - // https://github.com/grpc/grpc-node/issues/2647 - // https://github.com/googleapis/nodejs-firestore/issues/1023 - // https://github.com/googleapis/nodejs-firestore/issues/1023 - - return undefined; - })) - )?.[0]; - - yield cache; - - if (!cache) { - return; - } - - const age = Date.now() - cache.createdAt.valueOf(); - const stale = cache.createdAt.valueOf() < (Date.now() - cacheTolerance); - this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for ${urlToCrawl}, normalized digest: ${digest}, ${age}ms old, tolerance ${cacheTolerance}ms`, { - url: urlToCrawl, digest, age, stale, cacheTolerance - }); - - let snapshot: PageSnapshot | undefined; - let screenshotUrl: string | undefined; - let pageshotUrl: string | undefined; - const preparations = [ - this.firebaseObjectStorage.downloadFile(`snapshots/${cache._id}`).then((r) => { - snapshot = JSON.parse(r.toString('utf-8')); - }), - cache.screenshotAvailable ? - this.firebaseObjectStorage.signDownloadUrl(`screenshots/${cache._id}`, Date.now() + this.urlValidMs).then((r) => { - screenshotUrl = r; - }) : - Promise.resolve(undefined), - cache.pageshotAvailable ? - this.firebaseObjectStorage.signDownloadUrl(`pageshots/${cache._id}`, Date.now() + this.urlValidMs).then((r) => { - pageshotUrl = r; - }) : - Promise.resolve(undefined) - ]; - try { - await Promise.all(preparations); - } catch (_err) { - // Swallow cache errors. - return undefined; - } - - yield { - isFresh: !stale, - ...cache, - snapshot: { - ...snapshot, - screenshot: undefined, - pageshot: undefined, - screenshotUrl, - pageshotUrl, - } as PageSnapshot & { screenshotUrl?: string; pageshotUrl?: string; } - }; - } - - async setToCache(urlToCrawl: URL, snapshot: PageSnapshot) { - const digest = this.getUrlDigest(urlToCrawl); - - this.logger.info(`Caching snapshot of ${urlToCrawl}...`, { url: urlToCrawl, digest, title: snapshot?.title, href: snapshot?.href }); - const nowDate = new Date(); - - const cache = Crawled.from({ - _id: randomUUID(), - url: urlToCrawl.toString(), - createdAt: nowDate, - expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs), - htmlSignificantlyModifiedByJs: snapshot.htmlSignificantlyModifiedByJs, - urlPathDigest: digest, - }); - - const savingOfSnapshot = this.firebaseObjectStorage.saveFile(`snapshots/${cache._id}`, - Buffer.from( - JSON.stringify({ - ...snapshot, - screenshot: undefined, - pageshot: undefined, - }), - 'utf-8' - ), - { - metadata: { - contentType: 'application/json', - } + proxySnapshot.status = proxyLoaded.status; + proxySnapshot.statusText = proxyLoaded.statusText; + if ( + proxyLoaded.status === 200 && + crawlerOpts?.browserIsNotRequired() + ) { + } + analyzed = await this.jsdomControl.analyzeHTMLTextLite( + proxySnapshot.html, + ); + if (proxyLoaded.status === 200 || analyzed.tokens >= 200) { + proxySnapshot.isIntermediate = true; + if (sideLoadSnapshotPermitted) { + yield this.jsdomControl.narrowSnapshot(proxySnapshot, crawlOpts); } - ).then((r) => { - cache.snapshotAvailable = true; - return r; - }); - - if (snapshot.screenshot) { - await this.firebaseObjectStorage.saveFile(`screenshots/${cache._id}`, snapshot.screenshot, { - metadata: { - contentType: 'image/png', - } - }); - cache.screenshotAvailable = true; - } - if (snapshot.pageshot) { - await this.firebaseObjectStorage.saveFile(`pageshots/${cache._id}`, snapshot.pageshot, { - metadata: { - contentType: 'image/png', - } - }); - cache.pageshotAvailable = true; - } - await savingOfSnapshot; - this.batchedCaches.push(cache); - // const r = await Crawled.save(cache.degradeForFireStore()).catch((err) => { - // this.logger.error(`Failed to save cache for ${urlToCrawl}`, { err: marshalErrorLike(err) }); - - // return undefined; - // }); - - return cache; - } - - async *iterSnapshots(urlToCrawl: URL, crawlOpts?: ExtraScrappingOptions, crawlerOpts?: CrawlerOptions) { - // if (crawlerOpts?.respondWith.includes(CONTENT_FORMAT.VLM)) { - // const finalBrowserSnapshot = await this.getFinalSnapshot(urlToCrawl, { - // ...crawlOpts, engine: ENGINE_TYPE.BROWSER - // }, crawlerOpts); - - // yield* this.lmControl.geminiFromBrowserSnapshot(finalBrowserSnapshot); - - // return; - // } - - if (crawlerOpts?.respondWith.includes(CONTENT_FORMAT.XREAD_LM)) { - const finalAutoSnapshot = await this.getFinalSnapshot(urlToCrawl, { - ...crawlOpts, - engine: crawlOpts?.engine || ENGINE_TYPE.AUTO, - }, CrawlerOptions.from({ - ...crawlerOpts, - respondWith: 'html', - })); - - if (!finalAutoSnapshot?.html) { - throw new AssertionFailureError(`Unexpected non HTML content for XreadLM: ${urlToCrawl}`); - } - - if (crawlerOpts?.instruction || crawlerOpts?.jsonSchema) { - const jsonSchema = crawlerOpts.jsonSchema ? JSON.stringify(crawlerOpts.jsonSchema, undefined, 2) : undefined; - yield* this.lmControl.xreadLMFromSnapshot(crawlerOpts.instruction, jsonSchema, finalAutoSnapshot); - - return; - } - - try { - yield* this.lmControl.xreadLMMarkdownFromSnapshot(finalAutoSnapshot); - } catch (err) { - if (err instanceof HTTPServiceError && err.status === 429) { - throw new ServiceNodeResourceDrainError(`Xread LM is at capacity, please try again later.`); - } - throw err; - } - - return; - } - - yield* this.cachedScrap(urlToCrawl, crawlOpts, crawlerOpts); - } - - async *cachedScrap(urlToCrawl: URL, crawlOpts?: ExtraScrappingOptions, crawlerOpts?: CrawlerOptions) { - if (crawlerOpts?.html) { - const snapshot = { - href: urlToCrawl.toString(), - html: crawlerOpts.html, - title: '', - text: '', - } as PageSnapshot; - yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts); - - return; - } - - if (crawlerOpts?.pdf) { - const pdfFile = crawlerOpts.pdf instanceof FancyFile ? crawlerOpts.pdf : this.tempFileManager.cacheBuffer(Buffer.from(crawlerOpts.pdf, 'base64')); - const pdfLocalPath = pathToFileURL((await pdfFile.filePath)); - const snapshot = { - href: urlToCrawl.toString(), - html: ``, - title: '', - text: '', - pdfs: [pdfLocalPath.href], - } as PageSnapshot; - - yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts); - - return; + sideLoaded = proxyLoaded; + fallbackProxyIsUsed = true; + } } if ( - crawlOpts?.engine === ENGINE_TYPE.CURL || - // deprecated name - crawlOpts?.engine === 'direct' + crawlOpts && + (sideLoaded.status === 200 || + analyzed.tokens >= 200 || + crawlOpts.allocProxy) ) { - let sideLoaded; - try { - sideLoaded = (crawlOpts?.allocProxy && !crawlOpts?.proxyUrl) ? - await this.sideLoadWithAllocatedProxy(urlToCrawl, crawlOpts) : - await this.curlControl.sideLoad(urlToCrawl, crawlOpts); - - } catch (err) { - if (err instanceof ServiceBadAttemptError) { - throw new AssertionFailureError(err.message); - } - throw err; - } - if (!sideLoaded?.file) { - throw new AssertionFailureError(`Remote server did not return a body: ${urlToCrawl}`); - } - const draftSnapshot = await this.snapshotFormatter.createSnapshotFromFile(urlToCrawl, sideLoaded.file, sideLoaded.contentType, sideLoaded.fileName); - draftSnapshot.status = sideLoaded.status; - draftSnapshot.statusText = sideLoaded.statusText; - yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts); - return; + this.logger.info(`Side load seems to work, applying to crawler.`, { + url: urlToCrawl.href, + }); + crawlOpts.sideLoad ??= sideLoaded.sideLoadOpts; + if (fallbackProxyIsUsed) { + this.logger.info(`Proxy seems to salvage the page`, { + url: urlToCrawl.href, + }); + } } - if (crawlOpts?.engine === ENGINE_TYPE.CF_BROWSER_RENDERING) { - const html = await this.cfBrowserRendering.fetchContent(urlToCrawl.href); - const snapshot = { - href: urlToCrawl.toString(), - html, - title: '', - text: '', - } as PageSnapshot; - yield this.jsdomControl.narrowSnapshot(snapshot, crawlOpts); - return; - } - - const cacheTolerance = crawlerOpts?.cacheTolerance ?? this.cacheValidMs; - const cacheIt = this.queryCache(urlToCrawl, cacheTolerance); - - let cache = (await cacheIt.next()).value; - if (cache?.htmlSignificantlyModifiedByJs === false) { - if (crawlerOpts && crawlerOpts.timeout === undefined) { - crawlerOpts.respondTiming ??= RESPOND_TIMING.HTML; - } - } - - if (!crawlerOpts || crawlerOpts.isCacheQueryApplicable()) { - cache = (await cacheIt.next()).value; - } - cacheIt.return(undefined); - - if (cache?.isFresh && - (!crawlOpts?.favorScreenshot || (crawlOpts?.favorScreenshot && (cache.screenshotAvailable && cache.pageshotAvailable))) && - (_.get(cache.snapshot, 'locale') === crawlOpts?.locale) - ) { - if (cache.snapshot) { - cache.snapshot.isFromCache = true; - } - yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts); - - return; - } - - if (crawlOpts?.engine !== ENGINE_TYPE.BROWSER && !this.knownUrlThatSideLoadingWouldCrashTheBrowser(urlToCrawl)) { - const sideLoadSnapshotPermitted = crawlerOpts?.browserIsNotRequired() && - [RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes(crawlerOpts.presumedRespondTiming); - try { - const altOpts = { ...crawlOpts }; - let sideLoaded = (crawlOpts?.allocProxy && !crawlOpts?.proxyUrl) ? - await this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts) : - await this.curlControl.sideLoad(urlToCrawl, altOpts).catch((err) => { - this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { err: marshalErrorLike(err), href: urlToCrawl.href }); - - if (err instanceof ApplicationError && !(err instanceof ServiceBadAttemptError)) { - return Promise.reject(err); - } - - return this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts); - }); - if (!sideLoaded.file) { - throw new ServiceBadAttemptError(`Remote server did not return a body: ${urlToCrawl}`); - } - const draftSnapshot = await this.snapshotFormatter.createSnapshotFromFile( - urlToCrawl, sideLoaded.file, sideLoaded.contentType, sideLoaded.fileName - ).catch((err) => { - if (err instanceof ApplicationError) { - return Promise.reject(new ServiceBadAttemptError(err.message)); - } - return Promise.reject(err); - }); - draftSnapshot.status = sideLoaded.status; - draftSnapshot.statusText = sideLoaded.statusText; - if (sideLoaded.status == 200 && !sideLoaded.contentType.startsWith('text/html')) { - yield draftSnapshot; - return; - } - - let analyzed = await this.jsdomControl.analyzeHTMLTextLite(draftSnapshot.html); - draftSnapshot.title ??= analyzed.title; - draftSnapshot.isIntermediate = true; - if (sideLoadSnapshotPermitted) { - yield this.jsdomControl.narrowSnapshot(draftSnapshot, crawlOpts); - } - let fallbackProxyIsUsed = false; - if ( - ((!crawlOpts?.allocProxy || crawlOpts.allocProxy !== 'none') && !crawlOpts?.proxyUrl) && - (analyzed.tokens < 42 || sideLoaded.status !== 200) - ) { - const proxyLoaded = await this.sideLoadWithAllocatedProxy(urlToCrawl, altOpts); - if (!proxyLoaded.file) { - throw new ServiceBadAttemptError(`Remote server did not return a body: ${urlToCrawl}`); - } - const proxySnapshot = await this.snapshotFormatter.createSnapshotFromFile( - urlToCrawl, proxyLoaded.file, proxyLoaded.contentType, proxyLoaded.fileName - ).catch((err) => { - if (err instanceof ApplicationError) { - return Promise.reject(new ServiceBadAttemptError(err.message)); - } - return Promise.reject(err); - }); - proxySnapshot.status = proxyLoaded.status; - proxySnapshot.statusText = proxyLoaded.statusText; - if (proxyLoaded.status === 200 && crawlerOpts?.browserIsNotRequired()) { - } - analyzed = await this.jsdomControl.analyzeHTMLTextLite(proxySnapshot.html); - if (proxyLoaded.status === 200 || analyzed.tokens >= 200) { - proxySnapshot.isIntermediate = true; - if (sideLoadSnapshotPermitted) { - yield this.jsdomControl.narrowSnapshot(proxySnapshot, crawlOpts); - } - sideLoaded = proxyLoaded; - fallbackProxyIsUsed = true; - } - } - - if (crawlOpts && (sideLoaded.status === 200 || analyzed.tokens >= 200 || crawlOpts.allocProxy)) { - this.logger.info(`Side load seems to work, applying to crawler.`, { url: urlToCrawl.href }); - crawlOpts.sideLoad ??= sideLoaded.sideLoadOpts; - if (fallbackProxyIsUsed) { - this.logger.info(`Proxy seems to salvage the page`, { url: urlToCrawl.href }); - } - } - } catch (err: any) { - this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { err: marshalErrorLike(err), href: urlToCrawl.href }); - if (err instanceof ApplicationError && - !(err instanceof ServiceBadAttemptError) && - !(err instanceof DataStreamBrokenError) - ) { - throw err; - } - } - } else if (crawlOpts?.allocProxy && crawlOpts.allocProxy !== 'none' && !crawlOpts.proxyUrl) { - const proxyUrl = await this.proxyProvider.alloc(this.figureOutBestProxyCountry(crawlOpts)); - crawlOpts.proxyUrl = proxyUrl.href; - } - - try { - if (crawlOpts?.targetSelector || crawlOpts?.removeSelector || crawlOpts?.withIframe || crawlOpts?.withShadowDom) { - for await (const x of this.puppeteerControl.scrap(urlToCrawl, crawlOpts)) { - yield this.jsdomControl.narrowSnapshot(x, crawlOpts); - } - - return; - } - - yield* this.puppeteerControl.scrap(urlToCrawl, crawlOpts); - } catch (err: any) { - if (cache && !(err instanceof SecurityCompromiseError)) { - this.logger.warn(`Failed to scrap ${urlToCrawl}, but a stale cache is available. Falling back to cache`, { err: marshalErrorLike(err) }); - yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts); - return; - } - throw err; - } - } - - assignChargeAmount(formatted: FormattedPage, saasTierPolicy?: Parameters[0]) { - if (!formatted) { - return 0; - } - - let amount = 0; - if (formatted.content) { - amount = estimateToken(formatted.content); - } else if (formatted.description) { - amount += estimateToken(formatted.description); - } - - if (formatted.text) { - amount += estimateToken(formatted.text); - } - - if (formatted.html) { - amount += estimateToken(formatted.html); - } - if (formatted.screenshotUrl || formatted.screenshot) { - // OpenAI image token count for 1024x1024 image - amount += 765; - } - - if (saasTierPolicy) { - amount = this.saasApplyTierPolicy(saasTierPolicy, amount); - } - - Object.assign(formatted, { usage: { tokens: amount } }); - assignMeta(formatted, { usage: { tokens: amount } }); - - return amount; - } - - - async *scrapMany(urls: URL[], options?: ExtraScrappingOptions, crawlerOpts?: CrawlerOptions) { - const iterators = urls.map((url) => this.cachedScrap(url, options, crawlerOpts)); - - const results: (PageSnapshot | undefined)[] = iterators.map((_x) => undefined); - - let nextDeferred = Defer(); - let concluded = false; - - const handler = async (it: AsyncGenerator, idx: number) => { - try { - for await (const x of it) { - results[idx] = x; - - if (x) { - nextDeferred.resolve(); - nextDeferred = Defer(); - } - - } - } catch (err: any) { - this.logger.warn(`Failed to scrap ${urls[idx]}`, { err: marshalErrorLike(err) }); - } - }; - - Promise.allSettled( - iterators.map((it, idx) => handler(it, idx)) - ).finally(() => { - concluded = true; - nextDeferred.resolve(); + } catch (err: any) { + this.logger.warn(`Failed to side load ${urlToCrawl.origin}`, { + err: marshalErrorLike(err), + href: urlToCrawl.href, }); + if ( + err instanceof ApplicationError && + !(err instanceof ServiceBadAttemptError) && + !(err instanceof DataStreamBrokenError) + ) { + throw err; + } + } + } else if ( + crawlOpts?.allocProxy && + crawlOpts.allocProxy !== "none" && + !crawlOpts.proxyUrl + ) { + const proxyUrl = await this.proxyProvider.alloc( + this.figureOutBestProxyCountry(crawlOpts), + ); + crawlOpts.proxyUrl = proxyUrl.href; + } + + try { + if ( + crawlOpts?.targetSelector || + crawlOpts?.removeSelector || + crawlOpts?.withIframe || + crawlOpts?.withShadowDom + ) { + for await (const x of this.puppeteerControl.scrap( + urlToCrawl, + crawlOpts, + )) { + yield this.jsdomControl.narrowSnapshot(x, crawlOpts); + } + + return; + } + + yield* this.puppeteerControl.scrap(urlToCrawl, crawlOpts); + } catch (err: any) { + if (cache && !(err instanceof SecurityCompromiseError)) { + this.logger.warn( + `Failed to scrap ${urlToCrawl}, but a stale cache is available. Falling back to cache`, + { err: marshalErrorLike(err) }, + ); + yield this.jsdomControl.narrowSnapshot(cache.snapshot, crawlOpts); + return; + } + throw err; + } + } + + assignChargeAmount( + formatted: FormattedPage, + saasTierPolicy?: Parameters[0], + ) { + if (!formatted) { + return 0; + } + + let amount = 0; + if (formatted.content) { + amount = estimateToken(formatted.content); + } else if (formatted.description) { + amount += estimateToken(formatted.description); + } + + if (formatted.text) { + amount += estimateToken(formatted.text); + } + + if (formatted.html) { + amount += estimateToken(formatted.html); + } + if (formatted.screenshotUrl || formatted.screenshot) { + // OpenAI image token count for 1024x1024 image + amount += 765; + } + + if (saasTierPolicy) { + amount = this.saasApplyTierPolicy(saasTierPolicy, amount); + } + + Object.assign(formatted, { usage: { tokens: amount } }); + assignMeta(formatted, { usage: { tokens: amount } }); + + return amount; + } + + async *scrapMany( + urls: URL[], + options?: ExtraScrappingOptions, + crawlerOpts?: CrawlerOptions, + ) { + const iterators = urls.map((url) => + this.cachedScrap(url, options, crawlerOpts), + ); + + const results: (PageSnapshot | undefined)[] = iterators.map( + (_x) => undefined, + ); + + let nextDeferred = Defer(); + let concluded = false; + + const handler = async ( + it: AsyncGenerator, + idx: number, + ) => { + try { + for await (const x of it) { + results[idx] = x; + + if (x) { + nextDeferred.resolve(); + nextDeferred = Defer(); + } + } + } catch (err: any) { + this.logger.warn(`Failed to scrap ${urls[idx]}`, { + err: marshalErrorLike(err), + }); + } + }; + + Promise.allSettled(iterators.map((it, idx) => handler(it, idx))).finally( + () => { + concluded = true; + nextDeferred.resolve(); + }, + ); + + yield results; + + try { + while (!concluded) { + await nextDeferred.promise; yield results; + } + yield results; + } finally { + for (const x of iterators) { + x.return(); + } + } + } - try { - while (!concluded) { - await nextDeferred.promise; - - yield results; - } - yield results; - } finally { - for (const x of iterators) { - x.return(); - } - } + async configure(opts: CrawlerOptions) { + this.threadLocal.set("withGeneratedAlt", opts.withGeneratedAlt); + this.threadLocal.set("withLinksSummary", opts.withLinksSummary); + this.threadLocal.set("withImagesSummary", opts.withImagesSummary); + this.threadLocal.set("keepImgDataUrl", opts.keepImgDataUrl); + this.threadLocal.set("cacheTolerance", opts.cacheTolerance); + this.threadLocal.set("withIframe", opts.withIframe); + this.threadLocal.set("withShadowDom", opts.withShadowDom); + this.threadLocal.set("userAgent", opts.userAgent); + if (opts.timeout) { + this.threadLocal.set("timeout", opts.timeout * 1000); + } + this.threadLocal.set("retainImages", opts.retainImages); + this.threadLocal.set("noGfm", opts.noGfm); + this.threadLocal.set("DNT", Boolean(opts.doNotTrack)); + if (opts.markdown) { + this.threadLocal.set("turndownOpts", opts.markdown); } - async configure(opts: CrawlerOptions) { + const crawlOpts: ExtraScrappingOptions = { + proxyUrl: opts.proxyUrl, + cookies: opts.setCookies, + favorScreenshot: ["screenshot", "pageshot"].some((x) => + opts.respondWith.includes(x), + ), + removeSelector: opts.removeSelector, + targetSelector: opts.targetSelector, + waitForSelector: opts.waitForSelector, + overrideUserAgent: opts.userAgent, + timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined, + withIframe: opts.withIframe, + withShadowDom: opts.withShadowDom, + locale: opts.locale, + referer: opts.referer, + viewport: opts.viewport, + engine: opts.engine, + allocProxy: opts.proxy?.endsWith("+") + ? opts.proxy.slice(0, -1) + : opts.proxy, + proxyResources: opts.proxyUrl || opts.proxy?.endsWith("+") ? true : false, + private: Boolean(opts.doNotTrack), + }; - this.threadLocal.set('withGeneratedAlt', opts.withGeneratedAlt); - this.threadLocal.set('withLinksSummary', opts.withLinksSummary); - this.threadLocal.set('withImagesSummary', opts.withImagesSummary); - this.threadLocal.set('keepImgDataUrl', opts.keepImgDataUrl); - this.threadLocal.set('cacheTolerance', opts.cacheTolerance); - this.threadLocal.set('withIframe', opts.withIframe); - this.threadLocal.set('withShadowDom', opts.withShadowDom); - this.threadLocal.set('userAgent', opts.userAgent); - if (opts.timeout) { - this.threadLocal.set('timeout', opts.timeout * 1000); - } - this.threadLocal.set('retainImages', opts.retainImages); - this.threadLocal.set('noGfm', opts.noGfm); - this.threadLocal.set('DNT', Boolean(opts.doNotTrack)); - if (opts.markdown) { - this.threadLocal.set('turndownOpts', opts.markdown); - } - - const crawlOpts: ExtraScrappingOptions = { - proxyUrl: opts.proxyUrl, - cookies: opts.setCookies, - favorScreenshot: ['screenshot', 'pageshot'].some((x) => opts.respondWith.includes(x)), - removeSelector: opts.removeSelector, - targetSelector: opts.targetSelector, - waitForSelector: opts.waitForSelector, - overrideUserAgent: opts.userAgent, - timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined, - withIframe: opts.withIframe, - withShadowDom: opts.withShadowDom, - locale: opts.locale, - referer: opts.referer, - viewport: opts.viewport, - engine: opts.engine, - allocProxy: opts.proxy?.endsWith('+') ? opts.proxy.slice(0, -1) : opts.proxy, - proxyResources: (opts.proxyUrl || opts.proxy?.endsWith('+')) ? true : false, - private: Boolean(opts.doNotTrack), - }; - - if (crawlOpts.targetSelector?.length) { - if (typeof crawlOpts.targetSelector === 'string') { - crawlOpts.targetSelector = [crawlOpts.targetSelector]; - } - for (const s of crawlOpts.targetSelector) { - for (const e of s.split(',').map((x) => x.trim())) { - if (e.startsWith('*') || e.startsWith(':') || e.includes('*:')) { - throw new ParamValidationError({ - message: `Unacceptable selector: '${e}'. We cannot accept match-all selector for performance reasons. Sorry.`, - path: 'targetSelector' - }); - } - } - } - } - - if (opts._hintIps?.length) { - const hints = await this.geoIpService.lookupCities(opts._hintIps); - const board: Record = {}; - for (const x of hints) { - if (x.country?.code) { - board[x.country.code] = (board[x.country.code] || 0) + 1; - } - } - const hintCountry = _.maxBy(Array.from(Object.entries(board)), 1)?.[0]; - crawlOpts.countryHint = hintCountry?.toLowerCase(); - } - - if (opts.locale) { - crawlOpts.extraHeaders ??= {}; - crawlOpts.extraHeaders['Accept-Language'] = opts.locale; - } - - if (opts.respondWith.includes(CONTENT_FORMAT.VLM)) { - crawlOpts.favorScreenshot = true; - } - - if (opts.injectFrameScript?.length) { - crawlOpts.injectFrameScripts = (await Promise.all( - opts.injectFrameScript.map((x) => { - if (URL.canParse(x)) { - return fetch(x).then((r) => r.text()); - } - - return x; - }) - )).filter(Boolean); - } - - if (opts.injectPageScript?.length) { - crawlOpts.injectPageScripts = (await Promise.all( - opts.injectPageScript.map((x) => { - if (URL.canParse(x)) { - return fetch(x).then((r) => r.text()); - } - - return x; - }) - )).filter(Boolean); - } - - return crawlOpts; - } - - protected async formatSnapshot( - crawlerOptions: CrawlerOptions, - snapshot: PageSnapshot & { - screenshotUrl?: string; - pageshotUrl?: string; - }, - nominalUrl?: URL, - urlValidMs?: number, - scrappingOptions?: ScrappingOptions - ) { - const presumedURL = crawlerOptions.base === 'final' ? new URL(snapshot.href) : nominalUrl; - - const respondWith = crawlerOptions.respondWith; - if (respondWith === CONTENT_FORMAT.XREAD_LM || respondWith === CONTENT_FORMAT.VLM) { - const output: FormattedPage = { - title: snapshot.title, - content: snapshot.parsed?.textContent, - url: presumedURL?.href || snapshot.href, - }; - - Object.defineProperty(output, 'textRepresentation', { - value: snapshot.parsed?.textContent, - enumerable: false, + if (crawlOpts.targetSelector?.length) { + if (typeof crawlOpts.targetSelector === "string") { + crawlOpts.targetSelector = [crawlOpts.targetSelector]; + } + for (const s of crawlOpts.targetSelector) { + for (const e of s.split(",").map((x) => x.trim())) { + if (e.startsWith("*") || e.startsWith(":") || e.includes("*:")) { + throw new ParamValidationError({ + message: `Unacceptable selector: '${e}'. We cannot accept match-all selector for performance reasons. Sorry.`, + path: "targetSelector", }); - - return output; + } } - - return this.formatSnapshotWithPDFSideLoad(respondWith, snapshot, presumedURL, urlValidMs, scrappingOptions); + } } - async formatSnapshotWithPDFSideLoad(mode: string, snapshot: PageSnapshot, nominalUrl?: URL, urlValidMs?: number, scrappingOptions?: ScrappingOptions) { - const snapshotCopy = _.cloneDeep(snapshot); - - if (snapshotCopy.pdfs?.length) { - const pdfUrl = snapshotCopy.pdfs[0]; - if (pdfUrl.startsWith('http')) { - const sideLoaded = scrappingOptions?.sideLoad?.impersonate[pdfUrl]; - if (sideLoaded?.status === 200 && sideLoaded.body) { - snapshotCopy.pdfs[0] = pathToFileURL(await sideLoaded?.body.filePath).href; - return this.snapshotFormatter.formatSnapshot(mode, snapshotCopy, nominalUrl, urlValidMs); - } - - const r = await this.curlControl.sideLoad(new URL(pdfUrl), scrappingOptions).catch((err) => { - if (err instanceof ServiceBadAttemptError) { - return Promise.reject(new AssertionFailureError(`Failed to load PDF(${pdfUrl}): ${err.message}`)); - } - - return Promise.reject(err); - }); - if (r.status !== 200) { - throw new AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server responded status ${r.status}`); - } - if (!r.contentType.includes('application/pdf')) { - throw new AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server responded with wrong content type ${r.contentType}`); - } - if (!r.file) { - throw new AssertionFailureError(`Failed to load PDF(${pdfUrl}): Server did not return a body`); - } - snapshotCopy.pdfs[0] = pathToFileURL(await r.file.filePath).href; - } + if (opts._hintIps?.length) { + const hints = await this.geoIpService.lookupCities(opts._hintIps); + const board: Record = {}; + for (const x of hints) { + if (x.country?.code) { + board[x.country.code] = (board[x.country.code] || 0) + 1; } - - return this.snapshotFormatter.formatSnapshot(mode, snapshotCopy, nominalUrl, urlValidMs); + } + const hintCountry = _.maxBy(Array.from(Object.entries(board)), 1)?.[0]; + crawlOpts.countryHint = hintCountry?.toLowerCase(); } - async getFinalSnapshot(url: URL, opts?: ExtraScrappingOptions, crawlerOptions?: CrawlerOptions): Promise { - const it = this.cachedScrap(url, opts, crawlerOptions); - - let lastSnapshot; - let lastError; - try { - for await (const x of it) { - lastSnapshot = x; - } - } catch (err) { - lastError = err; - } - - if (!lastSnapshot && lastError) { - throw lastError; - } - - if (!lastSnapshot) { - throw new AssertionFailureError(`No content available`); - } - - return lastSnapshot; + if (opts.locale) { + crawlOpts.extraHeaders ??= {}; + crawlOpts.extraHeaders["Accept-Language"] = opts.locale; } - async simpleCrawl(mode: string, url: URL, opts?: ExtraScrappingOptions) { - const it = this.iterSnapshots(url, { ...opts, minIntervalMs: 500 }); + if (opts.respondWith.includes(CONTENT_FORMAT.VLM)) { + crawlOpts.favorScreenshot = true; + } - let lastSnapshot; - let goodEnough = false; - try { - for await (const x of it) { - lastSnapshot = x; - - if (goodEnough) { - break; - } - - if (lastSnapshot?.parsed?.content) { - // After it's good enough, wait for next snapshot; - goodEnough = true; - } + if (opts.injectFrameScript?.length) { + crawlOpts.injectFrameScripts = ( + await Promise.all( + opts.injectFrameScript.map((x) => { + if (URL.canParse(x)) { + return fetch(x).then((r) => r.text()); } - } catch (err) { - if (lastSnapshot) { - return this.snapshotFormatter.formatSnapshot(mode, lastSnapshot, url, this.urlValidMs); + return x; + }), + ) + ).filter(Boolean); + } + + if (opts.injectPageScript?.length) { + crawlOpts.injectPageScripts = ( + await Promise.all( + opts.injectPageScript.map((x) => { + if (URL.canParse(x)) { + return fetch(x).then((r) => r.text()); } - throw err; - } - - if (!lastSnapshot) { - throw new AssertionFailureError(`No content available`); - } - - return this.snapshotFormatter.formatSnapshot(mode, lastSnapshot, url, this.urlValidMs); + return x; + }), + ) + ).filter(Boolean); } - getDomainProfileUrlDigest(url: URL) { - const pathname = url.pathname; - const pathVec = pathname.split('/'); - const parentPath = pathVec.slice(0, -1).join('/'); + return crawlOpts; + } - const finalPath = parentPath || pathname; + protected async formatSnapshot( + crawlerOptions: CrawlerOptions, + snapshot: PageSnapshot & { + screenshotUrl?: string; + pageshotUrl?: string; + }, + nominalUrl?: URL, + urlValidMs?: number, + scrappingOptions?: ScrappingOptions, + ) { + const presumedURL = + crawlerOptions.base === "final" ? new URL(snapshot.href) : nominalUrl; - const key = url.origin.toLocaleLowerCase() + finalPath; + const respondWith = crawlerOptions.respondWith; + if ( + respondWith === CONTENT_FORMAT.XREAD_LM || + respondWith === CONTENT_FORMAT.VLM + ) { + const output: FormattedPage = { + title: snapshot.title, + content: snapshot.parsed?.textContent, + url: presumedURL?.href || snapshot.href, + }; - return { - digest: md5Hasher.hash(key), - path: finalPath, - }; + Object.defineProperty(output, "textRepresentation", { + value: snapshot.parsed?.textContent, + enumerable: false, + }); + + return output; } - proxyIterMap = new WeakMap>(); - @retryWith((err) => { - if (err instanceof ServiceBadApproachError) { - return false; + return this.formatSnapshotWithPDFSideLoad( + respondWith, + snapshot, + presumedURL, + urlValidMs, + scrappingOptions, + ); + } + + async formatSnapshotWithPDFSideLoad( + mode: string, + snapshot: PageSnapshot, + nominalUrl?: URL, + urlValidMs?: number, + scrappingOptions?: ScrappingOptions, + ) { + const snapshotCopy = _.cloneDeep(snapshot); + + if (snapshotCopy.pdfs?.length) { + const pdfUrl = snapshotCopy.pdfs[0]; + if (pdfUrl.startsWith("http")) { + const sideLoaded = scrappingOptions?.sideLoad?.impersonate[pdfUrl]; + if (sideLoaded?.status === 200 && sideLoaded.body) { + snapshotCopy.pdfs[0] = pathToFileURL( + await sideLoaded?.body.filePath, + ).href; + return this.snapshotFormatter.formatSnapshot( + mode, + snapshotCopy, + nominalUrl, + urlValidMs, + ); } - if (err instanceof ServiceBadAttemptError) { - // Keep trying - return true; - } - if (err instanceof ApplicationError) { - // Quit with this error - return false; - } - return undefined; - }, 3) - async sideLoadWithAllocatedProxy(url: URL, opts?: ExtraScrappingOptions) { - if (opts?.allocProxy === 'none') { - return this.curlControl.sideLoad(url, opts); - } - let proxy; - if (opts) { - let it = this.proxyIterMap.get(opts); - if (!it) { - it = this.proxyProvider.iterAlloc(this.figureOutBestProxyCountry(opts)); - this.proxyIterMap.set(opts, it); + + const r = await this.curlControl + .sideLoad(new URL(pdfUrl), scrappingOptions) + .catch((err) => { + if (err instanceof ServiceBadAttemptError) { + return Promise.reject( + new AssertionFailureError( + `Failed to load PDF(${pdfUrl}): ${err.message}`, + ), + ); } - proxy = (await it.next()).value; + + return Promise.reject(err); + }); + if (r.status !== 200) { + throw new AssertionFailureError( + `Failed to load PDF(${pdfUrl}): Server responded status ${r.status}`, + ); } - - proxy ??= await this.proxyProvider.alloc(this.figureOutBestProxyCountry(opts)); - this.logger.debug(`Proxy allocated`, { proxy: proxy.href }); - const r = await this.curlControl.sideLoad(url, { - ...opts, - proxyUrl: proxy.href, - }); - - if (opts && opts.allocProxy) { - opts.proxyUrl ??= proxy.href; + if (!r.contentType.includes("application/pdf")) { + throw new AssertionFailureError( + `Failed to load PDF(${pdfUrl}): Server responded with wrong content type ${r.contentType}`, + ); } - - return { ...r, proxy }; + if (!r.file) { + throw new AssertionFailureError( + `Failed to load PDF(${pdfUrl}): Server did not return a body`, + ); + } + snapshotCopy.pdfs[0] = pathToFileURL(await r.file.filePath).href; + } } - protected figureOutBestProxyCountry(opts?: ExtraScrappingOptions) { - if (!opts) { - return 'auto'; - } + return this.snapshotFormatter.formatSnapshot( + mode, + snapshotCopy, + nominalUrl, + urlValidMs, + ); + } - let draft; + async getFinalSnapshot( + url: URL, + opts?: ExtraScrappingOptions, + crawlerOptions?: CrawlerOptions, + ): Promise { + const it = this.cachedScrap(url, opts, crawlerOptions); - if (opts.allocProxy) { - if (this.proxyProvider.supports(opts.allocProxy)) { - draft = opts.allocProxy; - } else if (opts.allocProxy === 'none') { - return 'none'; - } - } - - if (opts.countryHint) { - if (this.proxyProvider.supports(opts.countryHint)) { - draft ??= opts.countryHint; - } - } - - draft ??= opts.allocProxy || 'auto'; - - return draft; + let lastSnapshot; + let lastError; + try { + for await (const x of it) { + lastSnapshot = x; + } + } catch (err) { + lastError = err; } - knownUrlThatSideLoadingWouldCrashTheBrowser(url: URL) { - if (url.hostname === 'chromewebstore.google.com') { - return true; - } - - return false; + if (!lastSnapshot && lastError) { + throw lastError; } - async saasAssertTierPolicy(opts: CrawlerOptions, auth: AuthDTO) { - let chargeScalar = 1; - let minimalCharge = 0; - - if (opts.withGeneratedAlt) { - await auth.assertTier(0, 'Alt text generation'); - minimalCharge = 765; - } - - if (opts.injectPageScript || opts.injectFrameScript) { - await auth.assertTier(0, 'Script injection'); - minimalCharge = 4_000; - } - - if (opts.withIframe) { - await auth.assertTier(0, 'Iframe'); - } - - if (opts.engine === ENGINE_TYPE.CF_BROWSER_RENDERING) { - await auth.assertTier(0, 'Cloudflare browser rendering'); - minimalCharge = 4_000; - } - - if (opts.respondWith.includes('lm') || opts.engine?.includes('lm')) { - await auth.assertTier(0, 'Language model'); - minimalCharge = 4_000; - chargeScalar = 3; - } - - if (opts.proxy && opts.proxy !== 'none') { - await auth.assertTier(['auto', 'any'].includes(opts.proxy) ? 0 : 2, 'Proxy allocation'); - chargeScalar = 5; - } - - return { - budget: opts.tokenBudget || 0, - chargeScalar, - minimalCharge, - }; + if (!lastSnapshot) { + throw new AssertionFailureError(`No content available`); } - saasApplyTierPolicy(policy: Awaited>, chargeAmount: number) { - const effectiveChargeAmount = policy.chargeScalar * Math.max(chargeAmount, policy.minimalCharge); - if (policy.budget && policy.budget < effectiveChargeAmount) { - throw new BudgetExceededError(`Token budget (${policy.budget}) exceeded, intended charge amount ${effectiveChargeAmount}`); + return lastSnapshot; + } + + async simpleCrawl(mode: string, url: URL, opts?: ExtraScrappingOptions) { + const it = this.iterSnapshots(url, { ...opts, minIntervalMs: 500 }); + + let lastSnapshot; + let goodEnough = false; + try { + for await (const x of it) { + lastSnapshot = x; + + if (goodEnough) { + break; } - return effectiveChargeAmount; + if (lastSnapshot?.parsed?.content) { + // After it's good enough, wait for next snapshot; + goodEnough = true; + } + } + } catch (err) { + if (lastSnapshot) { + return this.snapshotFormatter.formatSnapshot( + mode, + lastSnapshot, + url, + this.urlValidMs, + ); + } + + throw err; } + + if (!lastSnapshot) { + throw new AssertionFailureError(`No content available`); + } + + return this.snapshotFormatter.formatSnapshot( + mode, + lastSnapshot, + url, + this.urlValidMs, + ); + } + + getDomainProfileUrlDigest(url: URL) { + const pathname = url.pathname; + const pathVec = pathname.split("/"); + const parentPath = pathVec.slice(0, -1).join("/"); + + const finalPath = parentPath || pathname; + + const key = url.origin.toLocaleLowerCase() + finalPath; + + return { + digest: md5Hasher.hash(key), + path: finalPath, + }; + } + + proxyIterMap = new WeakMap< + ExtraScrappingOptions, + ReturnType + >(); + @retryWith((err) => { + if (err instanceof ServiceBadApproachError) { + return false; + } + if (err instanceof ServiceBadAttemptError) { + // Keep trying + return true; + } + if (err instanceof ApplicationError) { + // Quit with this error + return false; + } + return undefined; + }, 3) + async sideLoadWithAllocatedProxy(url: URL, opts?: ExtraScrappingOptions) { + if (opts?.allocProxy === "none") { + return this.curlControl.sideLoad(url, opts); + } + let proxy; + if (opts) { + let it = this.proxyIterMap.get(opts); + if (!it) { + it = this.proxyProvider.iterAlloc(this.figureOutBestProxyCountry(opts)); + this.proxyIterMap.set(opts, it); + } + proxy = (await it.next()).value; + } + + proxy ??= await this.proxyProvider.alloc( + this.figureOutBestProxyCountry(opts), + ); + this.logger.debug(`Proxy allocated`, { proxy: proxy.href }); + const r = await this.curlControl.sideLoad(url, { + ...opts, + proxyUrl: proxy.href, + }); + + if (opts && opts.allocProxy) { + opts.proxyUrl ??= proxy.href; + } + + return { ...r, proxy }; + } + + protected figureOutBestProxyCountry(opts?: ExtraScrappingOptions) { + if (!opts) { + return "auto"; + } + + let draft; + + if (opts.allocProxy) { + if (this.proxyProvider.supports(opts.allocProxy)) { + draft = opts.allocProxy; + } else if (opts.allocProxy === "none") { + return "none"; + } + } + + if (opts.countryHint) { + if (this.proxyProvider.supports(opts.countryHint)) { + draft ??= opts.countryHint; + } + } + + draft ??= opts.allocProxy || "auto"; + + return draft; + } + + knownUrlThatSideLoadingWouldCrashTheBrowser(url: URL) { + if (url.hostname === "chromewebstore.google.com") { + return true; + } + + return false; + } + + async saasAssertTierPolicy(opts: CrawlerOptions, auth: AuthDTO) { + let chargeScalar = 1; + let minimalCharge = 0; + + if (opts.withGeneratedAlt) { + await auth.assertTier(0, "Alt text generation"); + minimalCharge = 765; + } + + if (opts.injectPageScript || opts.injectFrameScript) { + await auth.assertTier(0, "Script injection"); + minimalCharge = 4_000; + } + + if (opts.withIframe) { + await auth.assertTier(0, "Iframe"); + } + + if (opts.engine === ENGINE_TYPE.CF_BROWSER_RENDERING) { + await auth.assertTier(0, "Cloudflare browser rendering"); + minimalCharge = 4_000; + } + + if (opts.respondWith.includes("lm") || opts.engine?.includes("lm")) { + await auth.assertTier(0, "Language model"); + minimalCharge = 4_000; + chargeScalar = 3; + } + + if (opts.proxy && opts.proxy !== "none") { + await auth.assertTier( + ["auto", "any"].includes(opts.proxy) ? 0 : 2, + "Proxy allocation", + ); + chargeScalar = 5; + } + + return { + budget: opts.tokenBudget || 0, + chargeScalar, + minimalCharge, + }; + } + + saasApplyTierPolicy( + policy: Awaited>, + chargeAmount: number, + ) { + const effectiveChargeAmount = + policy.chargeScalar * Math.max(chargeAmount, policy.minimalCharge); + if (policy.budget && policy.budget < effectiveChargeAmount) { + throw new BudgetExceededError( + `Token budget (${policy.budget}) exceeded, intended charge amount ${effectiveChargeAmount}`, + ); + } + + return effectiveChargeAmount; + } } diff --git a/src/api/searcher.ts b/src/api/searcher.ts index 235a0fc..74d6c00 100644 --- a/src/api/searcher.ts +++ b/src/api/searcher.ts @@ -1,940 +1,1152 @@ -import { singleton } from 'tsyringe'; +import { singleton } from "tsyringe"; import { - assignTransferProtocolMeta, RPCHost, RPCReflection, AssertionFailureError, assignMeta, RawString, -} from 'civkit/civ-rpc'; -import { marshalErrorLike } from 'civkit/lang'; -import { objHashMd5B64Of } from 'civkit/hash'; -import _ from 'lodash'; + assignTransferProtocolMeta, + RPCHost, + RPCReflection, + AssertionFailureError, + assignMeta, + RawString, +} from "civkit/civ-rpc"; +import { marshalErrorLike } from "civkit/lang"; +import { objHashMd5B64Of } from "civkit/hash"; +import _ from "lodash"; -import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit'; +import { + RateLimitControl, + RateLimitDesc, + RateLimitTriggeredError, +} from "../shared/services/rate-limit"; -import { CrawlerHost, ExtraScrappingOptions } from './crawler'; -import { CrawlerOptions, RESPOND_TIMING } from '../dto/crawler-options'; -import { SnapshotFormatter, FormattedPage as RealFormattedPage } from '../services/snapshot-formatter'; -import { GoogleSearchExplicitOperatorsDto } from '../services/serper-search'; +import { CrawlerHost, ExtraScrappingOptions } from "./crawler"; +import { CrawlerOptions, RESPOND_TIMING } from "../dto/crawler-options"; +import { + SnapshotFormatter, + FormattedPage as RealFormattedPage, +} from "../services/snapshot-formatter"; +import { GoogleSearchExplicitOperatorsDto } from "../services/serper-search"; -import { GlobalLogger } from '../services/logger'; -import { AsyncLocalContext } from '../services/async-context'; -import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry'; -import { OutputServerEventStream } from '../lib/transform-server-event-stream'; -import { ApiTokenAccount, AuthDTO } from '../dto/auth'; +import { GlobalLogger } from "../services/logger"; +import { AsyncLocalContext } from "../services/async-context"; +import { Context, Ctx, Method, Param, RPCReflect } from "../services/registry"; +import { OutputServerEventStream } from "../lib/transform-server-event-stream"; +import { ApiTokenAccount, AuthDTO } from "../dto/auth"; -import { GoogleSERP } from '../services/serp/google'; -import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper'; -import { toAsyncGenerator } from '../utils/misc'; -import { LRUCache } from 'lru-cache'; -import { API_CALL_STATUS } from '../shared/db/api-roll'; -import { SERPResult } from '../db/searched'; -import { SerperSearchQueryParams } from '../shared/3rd-party/serper-search'; -import { WebSearchEntry } from '../services/serp/compat'; -import { SecretExposer } from '../shared/services/secrets'; -import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback'; -import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; +import { GoogleSERP } from "../services/serp/google"; +import { + SerperBingSearchService, + SerperGoogleSearchService, +} from "../services/serp/serper"; +import { toAsyncGenerator } from "../utils/misc"; +import { LRUCache } from "lru-cache"; +import { API_CALL_STATUS } from "../shared/db/api-roll"; +import { SERPResult } from "../db/searched"; +import { SerperSearchQueryParams } from "../shared/3rd-party/serper-search"; +import { WebSearchEntry } from "../services/serp/compat"; +import { SecretExposer } from "../shared/services/secrets"; +import { StandaloneSearchFallbackService } from "../services/serp/standalone-fallback"; +import { STANDALONE_BOOT_TIMEOUT_MS } from "../services/boot-timeouts"; interface FormattedPage extends RealFormattedPage { - favicon?: string; - date?: string; + favicon?: string; + date?: string; } type RateLimitCache = { - blockedUntil?: Date; - user?: ApiTokenAccount; + blockedUntil?: Date; + user?: ApiTokenAccount; }; @singleton() export class SearcherHost extends RPCHost { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - cacheRetentionMs = 1000 * 3600 * 24 * 7; - cacheValidMs = 1000 * 3600; - pageCacheToleranceMs = 1000 * 3600 * 24; + cacheRetentionMs = 1000 * 3600 * 24 * 7; + cacheValidMs = 1000 * 3600; + pageCacheToleranceMs = 1000 * 3600 * 24; - reasonableDelayMs = 15_000; + reasonableDelayMs = 15_000; - targetResultCount = 5; + targetResultCount = 5; - highFreqKeyCache = new LRUCache({ - max: 256, - ttl: 60 * 60 * 1000, - updateAgeOnGet: false, - updateAgeOnHas: false, + highFreqKeyCache = new LRUCache({ + max: 256, + ttl: 60 * 60 * 1000, + updateAgeOnGet: false, + updateAgeOnHas: false, + }); + + batchedCaches: SERPResult[] = []; + + constructor( + protected globalLogger: GlobalLogger, + protected rateLimitControl: RateLimitControl, + protected threadLocal: AsyncLocalContext, + protected secretExposer: SecretExposer, + protected crawler: CrawlerHost, + protected snapshotFormatter: SnapshotFormatter, + protected googleSerp: GoogleSERP, + protected standaloneFallback: StandaloneSearchFallbackService, + protected serperGoogle: SerperGoogleSearchService, + protected serperBing: SerperBingSearchService, + ) { + super(...arguments); + + setInterval( + () => { + const thisBatch = this.batchedCaches; + this.batchedCaches = []; + if (!thisBatch.length) { + return; + } + const batch = SERPResult.DB.batch(); + + for (const x of thisBatch) { + batch.set(SERPResult.COLLECTION.doc(), x.degradeForFireStore()); + } + batch + .commit() + .then(() => { + this.logger.debug(`Saved ${thisBatch.length} caches by batch`); + }) + .catch((err) => { + this.logger.warn(`Failed to cache search result in batch`, { err }); + }); + }, + 1000 * 10 + Math.round(1000 * Math.random()), + ).unref(); + } + + override async init() { + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + + this.emit("ready"); + } + + override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { + return super.dependencyReady(timeoutMilliseconds); + } + + @Method({ + name: "searchIndex", + ext: { + http: { + action: ["get", "post"], + path: "/search", + }, + }, + tags: ["search"], + returnType: [String, OutputServerEventStream], + }) + @Method({ + ext: { + http: { + action: ["get", "post"], + path: "::q", + }, + }, + tags: ["search"], + returnType: [String, OutputServerEventStream, RawString], + }) + async search( + @RPCReflect() rpcReflect: RPCReflection, + @Ctx() ctx: Context, + auth: AuthDTO, + crawlerOptions: CrawlerOptions, + searchExplicitOperators: GoogleSearchExplicitOperatorsDto, + @Param("count", { validate: (v: number) => v >= 0 && v <= 20 }) + count: number, + @Param("type", { type: new Set(["web", "images", "news"]), default: "web" }) + variant: "web" | "images" | "news", + @Param("provider", { type: new Set(["google", "bing"]), default: "google" }) + searchEngine: "google" | "bing", + @Param("num", { validate: (v: number) => v >= 0 && v <= 20 }) + num?: number, + @Param("gl", { validate: (v: string) => /^[a-z]{2}$/i.test(v || "") }) + gl?: string, + @Param("hl", { validate: (v: string) => /^[a-z]{2}$/i.test(v || "") }) + hl?: string, + @Param("location") location?: string, + @Param("page") page?: number, + @Param("fallback", { type: Boolean, default: true }) fallback?: boolean, + @Param("q") q?: string, + ) { + // We want to make our search API follow SERP schema, so we need to expose 'num' parameter. + // Since we used 'count' as 'num' previously, we need to keep 'count' for old users. + // Here we combine 'count' and 'num' to 'count' for the rest of the function. + count = (num !== undefined ? num : count) ?? 10; + + const authToken = auth.bearerToken; + let highFreqKey: RateLimitCache | undefined; + if (authToken && this.highFreqKeyCache.has(authToken)) { + highFreqKey = this.highFreqKeyCache.get(authToken)!; + auth.user = highFreqKey.user; + auth.uid = highFreqKey.user?.user_id; + } + + const uid = await auth.solveUID(); + // Return content by default + const crawlWithoutContent = + crawlerOptions.respondWith.includes("no-content"); + const withFavicon = Boolean(ctx.get("X-With-Favicons")); + this.threadLocal.set("collect-favicon", withFavicon); + crawlerOptions.respondTiming ??= RESPOND_TIMING.VISIBLE_CONTENT; + + let chargeAmount = 0; + const noSlashPath = decodeURIComponent(ctx.path).slice(1); + if (!noSlashPath && !q) { + const index = await this.crawler.getIndex(ctx, auth); + if ( + !ctx.accepts("text/plain") && + (ctx.accepts("text/json") || ctx.accepts("application/json")) + ) { + return index; + } + + return assignTransferProtocolMeta(`${index}`, { + contentType: "text/plain", + envelope: null, + }); + } + + const user = await auth.assertUser(); + + if (highFreqKey?.blockedUntil) { + const now = new Date(); + const blockedTimeRemaining = + highFreqKey.blockedUntil.valueOf() - now.valueOf(); + if (blockedTimeRemaining > 0) { + throw RateLimitTriggeredError.from({ + message: `Per UID rate limit exceeded (async)`, + retryAfter: Math.ceil(blockedTimeRemaining / 1000), + }); + } + } + + const rateLimitPolicy = auth.getRateLimits( + rpcReflect.name.toUpperCase(), + ) || [ + parseInt(user.metadata?.speed_level) >= 2 + ? RateLimitDesc.from({ + occurrence: 1000, + periodSeconds: 60, + }) + : RateLimitDesc.from({ + occurrence: 100, + periodSeconds: 60, + }), + ]; + + const apiRollPromise = uid + ? this.rateLimitControl.simpleRPCUidBasedLimit( + rpcReflect, + uid, + [rpcReflect.name.toUpperCase()], + ...rateLimitPolicy, + ) + : this.rateLimitControl.simpleRpcIPBasedLimit( + rpcReflect, + ctx.ip || "anonymous", + [rpcReflect.name.toUpperCase()], + [[new Date(Date.now() - 60 * 1000), 20]], + ); + + if (uid && !highFreqKey) { + // Normal path + await apiRollPromise; + + if ( + rateLimitPolicy.some((x) => { + const rpm = x.occurrence / (x.periodSeconds / 60); + if (rpm >= 400) { + return true; + } + + return false; + }) + ) { + this.highFreqKeyCache.set(auth.bearerToken!, { + user, + }); + } + } else if (uid && highFreqKey) { + // High freq key path + apiRollPromise + .then( + // Rate limit not triggered, make sure not blocking. + () => { + delete highFreqKey.blockedUntil; + }, + // Rate limit triggered + (err) => { + if (!(err instanceof RateLimitTriggeredError)) { + return; + } + const now = Date.now(); + let tgtDate; + if (err.retryAfterDate) { + tgtDate = err.retryAfterDate; + } else if (err.retryAfter) { + tgtDate = new Date(now + err.retryAfter * 1000); + } + + if (tgtDate) { + const dt = tgtDate.valueOf() - now; + highFreqKey.blockedUntil = tgtDate; + setTimeout(() => { + if (highFreqKey.blockedUntil === tgtDate) { + delete highFreqKey.blockedUntil; + } + }, dt).unref(); + } + }, + ) + .finally(async () => { + // Always asynchronously update user(wallet); + const user = await auth.getBrief().catch(() => undefined); + if (user) { + highFreqKey.user = user; + } + }); + } + + rpcReflect.finally(async () => { + if (chargeAmount) { + if (uid) { + auth + .reportUsage(chargeAmount, `xread-${rpcReflect.name}`) + .catch((err) => { + this.logger.warn(`Unable to report usage for ${uid}`, { + err: marshalErrorLike(err), + }); + }); + } + try { + const apiRoll = await apiRollPromise; + apiRoll.chargeAmount = chargeAmount; + await apiRoll.save({ merge: true }); + } catch (_err) { + await this.rateLimitControl + .record({ + uid, + tags: [rpcReflect.name.toUpperCase()], + status: API_CALL_STATUS.SUCCESS, + chargeAmount, + }) + .save() + .catch((err) => { + this.logger.warn(`Failed to save rate limit record`, { + err: marshalErrorLike(err), + }); + }); + } + } }); - batchedCaches: SERPResult[] = []; + delete crawlerOptions.html; - constructor( - protected globalLogger: GlobalLogger, - protected rateLimitControl: RateLimitControl, - protected threadLocal: AsyncLocalContext, - protected secretExposer: SecretExposer, - protected crawler: CrawlerHost, - protected snapshotFormatter: SnapshotFormatter, - protected googleSerp: GoogleSERP, - protected standaloneFallback: StandaloneSearchFallbackService, - protected serperGoogle: SerperGoogleSearchService, - protected serperBing: SerperBingSearchService, + const crawlOpts = await this.crawler.configure(crawlerOptions); + const searchQuery = searchExplicitOperators.addTo(q || noSlashPath); + + let fetchNum = count; + if ((page ?? 1) === 1) { + fetchNum = count > 10 ? 30 : 20; + } + + let fallbackQuery: string | undefined; + let chargeAmountScaler = 1; + if (searchEngine === "bing") { + this.threadLocal.set("bing-preferred", true); + chargeAmountScaler = 3; + } + + if (variant !== "web") { + chargeAmountScaler = 5; + } + + // Search with fallback logic if enabled + const searchParams = { + variant, + provider: searchEngine, + q: searchQuery, + num: fetchNum, + gl, + hl, + location, + page, + }; + + const { + results, + query: successQuery, + tryTimes, + } = await this.searchWithFallback( + searchParams, + fallback, + crawlerOptions.noCache, + ); + chargeAmountScaler *= tryTimes; + + fallbackQuery = successQuery !== searchQuery ? successQuery : undefined; + + if (!results.length) { + throw new AssertionFailureError( + `No search results available for query ${searchQuery}`, + ); + } + + if (crawlOpts.timeoutMs && crawlOpts.timeoutMs < 30_000) { + delete crawlOpts.timeoutMs; + } + + let lastScrapped: any[] | undefined; + const targetResultCount = crawlWithoutContent ? count : count + 2; + const trimmedResults: any[] = results + .filter((x) => Boolean(x.link)) + .slice(0, targetResultCount) + .map((x) => this.mapToFinalResults(x)); + trimmedResults.toString = function () { + let r = + this.map((x, i) => (x ? Reflect.apply(x.toString, x, [i]) : "")) + .join("\n\n") + .trimEnd() + "\n"; + if (fallbackQuery) { + r = `Fallback query: ${fallbackQuery}\n\n${r}`; + } + return r; + }; + if ( + !crawlerOptions.respondWith.includes("no-content") && + ["html", "text", "shot", "markdown", "content"].some((x) => + crawlerOptions.respondWith.includes(x), + ) ) { - super(...arguments); + for (const x of trimmedResults) { + x.content ??= ""; + } + } + const assigningOfGeneralMixins = Promise.allSettled( + trimmedResults.map((x) => this.assignGeneralMixin(x)), + ); - setInterval(() => { - const thisBatch = this.batchedCaches; - this.batchedCaches = []; - if (!thisBatch.length) { - return; - } - const batch = SERPResult.DB.batch(); + let it; - for (const x of thisBatch) { - batch.set(SERPResult.COLLECTION.doc(), x.degradeForFireStore()); - } - batch.commit() - .then(() => { - this.logger.debug(`Saved ${thisBatch.length} caches by batch`); - }) - .catch((err) => { - this.logger.warn(`Failed to cache search result in batch`, { err }); - }); - }, 1000 * 10 + Math.round(1000 * Math.random())).unref(); + if (crawlWithoutContent || count === 0) { + it = toAsyncGenerator(trimmedResults); + await assigningOfGeneralMixins; + } else { + it = this.fetchSearchResults( + crawlerOptions.respondWith, + trimmedResults, + crawlOpts, + CrawlerOptions.from({ + ...crawlerOptions, + cacheTolerance: + crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs, + }), + count, + ); } - override async init() { - await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + if (!ctx.accepts("text/plain") && ctx.accepts("text/event-stream")) { + const sseStream = new OutputServerEventStream(); + rpcReflect.return(sseStream); + try { + for await (const scrapped of it) { + if (!scrapped) { + continue; + } + if (rpcReflect.signal.aborted) { + break; + } - this.emit('ready'); - } + chargeAmount = this.assignChargeAmount( + scrapped, + count, + chargeAmountScaler, + fallbackQuery, + ); + lastScrapped = scrapped; - override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { - return super.dependencyReady(timeoutMilliseconds); - } - - @Method({ - name: 'searchIndex', - ext: { - http: { - action: ['get', 'post'], - path: '/search' - } - }, - tags: ['search'], - returnType: [String, OutputServerEventStream], - }) - @Method({ - ext: { - http: { - action: ['get', 'post'], - path: '::q' - } - }, - tags: ['search'], - returnType: [String, OutputServerEventStream, RawString], - }) - async search( - @RPCReflect() rpcReflect: RPCReflection, - @Ctx() ctx: Context, - auth: AuthDTO, - crawlerOptions: CrawlerOptions, - searchExplicitOperators: GoogleSearchExplicitOperatorsDto, - @Param('count', { validate: (v: number) => v >= 0 && v <= 20 }) - count: number, - @Param('type', { type: new Set(['web', 'images', 'news']), default: 'web' }) - variant: 'web' | 'images' | 'news', - @Param('provider', { type: new Set(['google', 'bing']), default: 'google' }) - searchEngine: 'google' | 'bing', - @Param('num', { validate: (v: number) => v >= 0 && v <= 20 }) - num?: number, - @Param('gl', { validate: (v: string) => /^[a-z]{2}$/i.test(v || '') }) gl?: string, - @Param('hl', { validate: (v: string) => /^[a-z]{2}$/i.test(v || '') }) hl?: string, - @Param('location') location?: string, - @Param('page') page?: number, - @Param('fallback', { type: Boolean, default: true }) fallback?: boolean, - @Param('q') q?: string, - ) { - // We want to make our search API follow SERP schema, so we need to expose 'num' parameter. - // Since we used 'count' as 'num' previously, we need to keep 'count' for old users. - // Here we combine 'count' and 'num' to 'count' for the rest of the function. - count = (num !== undefined ? num : count) ?? 10; - - const authToken = auth.bearerToken; - let highFreqKey: RateLimitCache | undefined; - if (authToken && this.highFreqKeyCache.has(authToken)) { - highFreqKey = this.highFreqKeyCache.get(authToken)!; - auth.user = highFreqKey.user; - auth.uid = highFreqKey.user?.user_id; - } - - const uid = await auth.solveUID(); - // Return content by default - const crawlWithoutContent = crawlerOptions.respondWith.includes('no-content'); - const withFavicon = Boolean(ctx.get('X-With-Favicons')); - this.threadLocal.set('collect-favicon', withFavicon); - crawlerOptions.respondTiming ??= RESPOND_TIMING.VISIBLE_CONTENT; - - let chargeAmount = 0; - const noSlashPath = decodeURIComponent(ctx.path).slice(1); - if (!noSlashPath && !q) { - const index = await this.crawler.getIndex(auth); - if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) { - - return index; - } - - return assignTransferProtocolMeta(`${index}`, - { contentType: 'text/plain', envelope: null } - ); - } - - const user = await auth.assertUser(); - - if (highFreqKey?.blockedUntil) { - const now = new Date(); - const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf()); - if (blockedTimeRemaining > 0) { - throw RateLimitTriggeredError.from({ - message: `Per UID rate limit exceeded (async)`, - retryAfter: Math.ceil(blockedTimeRemaining / 1000), - }); - } - } - - const rateLimitPolicy = auth.getRateLimits(rpcReflect.name.toUpperCase()) || [ - parseInt(user.metadata?.speed_level) >= 2 ? - RateLimitDesc.from({ - occurrence: 1000, - periodSeconds: 60 - }) : - RateLimitDesc.from({ - occurrence: 100, - periodSeconds: 60 - }) - ]; - - const apiRollPromise = uid ? - this.rateLimitControl.simpleRPCUidBasedLimit( - rpcReflect, uid, [rpcReflect.name.toUpperCase()], - ...rateLimitPolicy - ) : - this.rateLimitControl.simpleRpcIPBasedLimit( - rpcReflect, ctx.ip || 'anonymous', [rpcReflect.name.toUpperCase()], - [[new Date(Date.now() - 60 * 1000), 20]] - ); - - if (uid && !highFreqKey) { - // Normal path - await apiRollPromise; - - if (rateLimitPolicy.some( - (x) => { - const rpm = x.occurrence / (x.periodSeconds / 60); - if (rpm >= 400) { - return true; - } - - return false; - }) - ) { - this.highFreqKeyCache.set(auth.bearerToken!, { - user, - }); - } - - } else if (uid && highFreqKey) { - // High freq key path - apiRollPromise.then( - // Rate limit not triggered, make sure not blocking. - () => { - delete highFreqKey.blockedUntil; - }, - // Rate limit triggered - (err) => { - if (!(err instanceof RateLimitTriggeredError)) { - return; - } - const now = Date.now(); - let tgtDate; - if (err.retryAfterDate) { - tgtDate = err.retryAfterDate; - } else if (err.retryAfter) { - tgtDate = new Date(now + err.retryAfter * 1000); - } - - if (tgtDate) { - const dt = tgtDate.valueOf() - now; - highFreqKey.blockedUntil = tgtDate; - setTimeout(() => { - if (highFreqKey.blockedUntil === tgtDate) { - delete highFreqKey.blockedUntil; - } - }, dt).unref(); - } - } - ).finally(async () => { - // Always asynchronously update user(wallet); - const user = await auth.getBrief().catch(() => undefined); - if (user) { - highFreqKey.user = user; - } + if (fallbackQuery) { + sseStream.write({ + event: "meta", + data: { fallback: fallbackQuery }, }); + } + + sseStream.write({ + event: "data", + data: scrapped, + }); } + } catch (err: any) { + this.logger.error( + `Failed to collect search result for query ${searchQuery}`, + { err: marshalErrorLike(err) }, + ); + sseStream.write({ + event: "error", + data: marshalErrorLike(err), + }); + } - rpcReflect.finally(async () => { - if (chargeAmount) { - if (uid) { - auth.reportUsage(chargeAmount, `xread-${rpcReflect.name}`).catch((err) => { - this.logger.warn(`Unable to report usage for ${uid}`, { err: marshalErrorLike(err) }); - }); - } - try { - const apiRoll = await apiRollPromise; - apiRoll.chargeAmount = chargeAmount; - await apiRoll.save({ merge: true }); + sseStream.end(); - } catch (_err) { - await this.rateLimitControl.record({ - uid, - tags: [rpcReflect.name.toUpperCase()], - status: API_CALL_STATUS.SUCCESS, - chargeAmount, - }).save().catch((err) => { - this.logger.warn(`Failed to save rate limit record`, { err: marshalErrorLike(err) }); - }); - } + return sseStream; + } + + let earlyReturn = false; + if ( + !ctx.accepts("text/plain") && + (ctx.accepts("text/json") || ctx.accepts("application/json")) + ) { + let earlyReturnTimer: ReturnType | undefined; + const setEarlyReturnTimer = () => { + if (earlyReturnTimer) { + return; + } + earlyReturnTimer = setTimeout( + async () => { + if (!lastScrapped) { + return; } + await assigningOfGeneralMixins; + chargeAmount = this.assignChargeAmount( + lastScrapped, + count, + chargeAmountScaler, + fallbackQuery, + ); + + rpcReflect.return(lastScrapped); + earlyReturn = true; + }, + (crawlerOptions.timeout || 0) * 1000 || this.reasonableDelayMs, + ); + }; + + for await (const scrapped of it) { + lastScrapped = scrapped; + if (rpcReflect.signal.aborted || earlyReturn) { + break; + } + if (_.some(scrapped, (x) => this.pageQualified(x))) { + setEarlyReturnTimer(); + } + if (!this.searchResultsQualified(scrapped, count)) { + continue; + } + if (earlyReturnTimer) { + clearTimeout(earlyReturnTimer); + } + await assigningOfGeneralMixins; + chargeAmount = this.assignChargeAmount( + scrapped, + count, + chargeAmountScaler, + fallbackQuery, + ); + + return scrapped; + } + + if (earlyReturnTimer) { + clearTimeout(earlyReturnTimer); + } + + if (!lastScrapped) { + throw new AssertionFailureError( + `No content available for query ${searchQuery}`, + ); + } + + if (!earlyReturn) { + await assigningOfGeneralMixins; + chargeAmount = this.assignChargeAmount( + lastScrapped, + count, + chargeAmountScaler, + fallbackQuery, + ); + } + + return lastScrapped; + } + + let earlyReturnTimer: ReturnType | undefined; + const setEarlyReturnTimer = () => { + if (earlyReturnTimer) { + return; + } + earlyReturnTimer = setTimeout( + async () => { + if (!lastScrapped) { + return; + } + await assigningOfGeneralMixins; + chargeAmount = this.assignChargeAmount( + lastScrapped, + count, + chargeAmountScaler, + fallbackQuery, + ); + + rpcReflect.return( + assignTransferProtocolMeta(`${lastScrapped}`, { + contentType: "text/plain", + envelope: null, + }), + ); + earlyReturn = true; + }, + (crawlerOptions.timeout || 0) * 1000 || this.reasonableDelayMs, + ); + }; + + for await (const scrapped of it) { + lastScrapped = scrapped; + if (rpcReflect.signal.aborted || earlyReturn) { + break; + } + if (_.some(scrapped, (x) => this.pageQualified(x))) { + setEarlyReturnTimer(); + } + + if (!this.searchResultsQualified(scrapped, count)) { + continue; + } + + if (earlyReturnTimer) { + clearTimeout(earlyReturnTimer); + } + await assigningOfGeneralMixins; + chargeAmount = this.assignChargeAmount( + scrapped, + count, + chargeAmountScaler, + fallbackQuery, + ); + + return assignTransferProtocolMeta(`${scrapped}`, { + contentType: "text/plain", + envelope: null, + }); + } + + if (earlyReturnTimer) { + clearTimeout(earlyReturnTimer); + } + + if (!lastScrapped) { + throw new AssertionFailureError( + `No content available for query ${searchQuery}`, + ); + } + + if (!earlyReturn) { + await assigningOfGeneralMixins; + chargeAmount = this.assignChargeAmount( + lastScrapped, + count, + chargeAmountScaler, + fallbackQuery, + ); + } + + return assignTransferProtocolMeta(`${lastScrapped}`, { + contentType: "text/plain", + envelope: null, + }); + } + + /** + * Search with fallback to progressively shorter queries if no results found + * @param params Search parameters + * @param useFallback Whether to use the fallback mechanism + * @param noCache Whether to bypass cache + * @returns Search response and the successful query + */ + async searchWithFallback( + params: SerperSearchQueryParams & { + variant: "web" | "images" | "news"; + provider?: string; + }, + useFallback: boolean = false, + noCache: boolean = false, + ) { + // Try original query first + const originalQuery = params.q; + const containsRTL = + /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test( + originalQuery, + ); + + // Extract results based on variant + let tryTimes = 1; + const results = await this.cachedSearch(params.variant, params, noCache); + if (results.length || !useFallback) { + return { results, query: params.q, tryTimes }; + } + + let queryTerms = originalQuery.split(/\s+/); + const lastResort = containsRTL + ? queryTerms.slice(queryTerms.length - 2) + : queryTerms.slice(0, 2); + + this.logger.info( + `No results for "${originalQuery}", trying fallback queries`, + ); + + let terms: string[] = []; + // fallback n times + const n = 4; + + while (tryTimes < n) { + const delta = Math.ceil(queryTerms.length / n) * tryTimes; + terms = containsRTL + ? queryTerms.slice(delta) + : queryTerms.slice(0, queryTerms.length - delta); + const query = terms.join(" "); + if (!query) { + break; + } + tryTimes += 1; + this.logger.info(`Retrying search with fallback query: "${query}"`); + const fallbackParams = { ...params, q: query }; + const fallbackResults = await this.cachedSearch( + params.variant, + fallbackParams, + noCache, + ); + if (fallbackResults.length > 0) { + return { results: fallbackResults, query: fallbackParams.q, tryTimes }; + } + } + + if (terms.length > lastResort.length) { + const query = lastResort.join(" "); + this.logger.info(`Retrying search with fallback query: "${query}"`); + const fallbackParams = { ...params, q: query }; + tryTimes += 1; + const fallbackResults = await this.cachedSearch( + params.variant, + fallbackParams, + noCache, + ); + + if (fallbackResults.length > 0) { + return { results: fallbackResults, query, tryTimes }; + } + } + + return { results, query: originalQuery, tryTimes }; + } + + async *fetchSearchResults( + mode: + | string + | "markdown" + | "html" + | "text" + | "screenshot" + | "favicon" + | "content", + searchResults?: FormattedPage[], + options?: ExtraScrappingOptions, + crawlerOptions?: CrawlerOptions, + count?: number, + ) { + if (!searchResults) { + return; + } + const urls = searchResults.map((x) => new URL(x.url!)); + const snapshotMap = new WeakMap(); + for await (const scrapped of this.crawler.scrapMany( + urls, + options, + crawlerOptions, + )) { + const mapped = scrapped.map((x, i) => { + if (!x) { + return {}; + } + if (snapshotMap.has(x)) { + return snapshotMap.get(x); + } + return this.crawler + .formatSnapshotWithPDFSideLoad(mode, x, urls[i], undefined, options) + .then((r) => { + snapshotMap.set(x, r); + + return r; + }) + .catch((err) => { + this.logger.error(`Failed to format snapshot for ${urls[i].href}`, { + err: marshalErrorLike(err), + }); + + return {}; + }); + }); + + const resultArray = (await Promise.all(mapped)) as FormattedPage[]; + for (const [i, v] of resultArray.entries()) { + if (v) { + Object.assign(searchResults[i], v); + } + } + + yield this.reOrganizeSearchResults(searchResults, count); + } + } + + reOrganizeSearchResults(searchResults: FormattedPage[], count?: number) { + const targetResultCount = count || this.targetResultCount; + const [qualifiedPages, unqualifiedPages] = _.partition(searchResults, (x) => + this.pageQualified(x), + ); + const acceptSet = new Set(qualifiedPages); + + const n = targetResultCount - qualifiedPages.length; + for (const x of unqualifiedPages.slice(0, n >= 0 ? n : 0)) { + acceptSet.add(x); + } + + const filtered = searchResults + .filter((x) => acceptSet.has(x)) + .slice(0, targetResultCount); + + const resultArray = filtered; + + resultArray.toString = searchResults.toString; + + return resultArray; + } + + assignChargeAmount( + formatted: FormattedPage[], + num: number, + scaler: number, + fallbackQuery?: string, + ) { + let contentCharge = 0; + for (const x of formatted) { + const itemAmount = this.crawler.assignChargeAmount(x) || 0; + + if (!itemAmount) { + continue; + } + + contentCharge += itemAmount; + } + + const numCharge = Math.ceil(formatted.length / 10) * 10000 * scaler; + + const final = Math.max(contentCharge, numCharge); + + if (final === numCharge) { + for (const x of formatted) { + x.usage = { tokens: Math.ceil(numCharge / formatted.length) }; + } + } + + const metadata: Record = { usage: { tokens: final } }; + if (fallbackQuery) { + metadata.fallback = fallbackQuery; + } + + assignMeta(formatted, metadata); + + return final; + } + + pageQualified(formattedPage: FormattedPage) { + return ( + (formattedPage.title && formattedPage.content) || + formattedPage.screenshotUrl || + formattedPage.pageshotUrl || + formattedPage.text || + formattedPage.html + ); + } + + searchResultsQualified( + results: FormattedPage[], + targetResultCount = this.targetResultCount, + ) { + return ( + _.every(results, (x) => this.pageQualified(x)) && + results.length >= targetResultCount + ); + } + + async getFavicon(domain: string) { + const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`; + + try { + const response = await fetch(url); + if (!response.ok) { + return ""; + } + const ab = await response.arrayBuffer(); + const buffer = Buffer.from(ab); + const base64 = buffer.toString("base64"); + return `data:image/png;base64,${base64}`; + } catch (error: any) { + this.logger.warn(`Failed to get favicon base64 string`, { + err: marshalErrorLike(error), + }); + return ""; + } + } + + *iterProviders(preference?: string, _variant?: string) { + const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY; + const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY); + + if (preference === "bing") { + if (preferScraping) { + yield this.standaloneFallback; + } + if (includeSerper) { + yield this.serperBing; + yield this.serperGoogle; + } + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + } + + return; + } + + if (preference === "google") { + if (preferScraping) { + yield this.standaloneFallback; + yield this.googleSerp; + } + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + yield this.serperGoogle; + } + + return; + } + + if (preferScraping) { + yield this.standaloneFallback; + yield this.googleSerp; + } + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + yield this.serperGoogle; + yield this.serperBing; + } + } + + async cachedSearch( + variant: "web" | "news" | "images", + query: Record, + noCache?: boolean, + ): Promise { + const queryDigest = objHashMd5B64Of({ ...query, variant }); + const provider = query.provider; + Reflect.deleteProperty(query, "provider"); + let cache; + if (!noCache) { + cache = ( + await SERPResult.fromFirestoreQuery( + SERPResult.COLLECTION.where("queryDigest", "==", queryDigest) + .orderBy("createdAt", "desc") + .limit(1), + ) + )[0]; + if (cache) { + const age = Date.now() - cache.createdAt.valueOf(); + const stale = + cache.createdAt.valueOf() < Date.now() - this.cacheValidMs; + this.logger.info( + `${stale ? "Stale cache exists" : "Cache hit"} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`, + { + query, + digest: queryDigest, + age, + stale, + }, + ); + + if (!stale) { + return cache.response as any; + } + } + } + + try { + let r: any[] | undefined; + let lastError; + outerLoop: for (const client of this.iterProviders(provider, variant)) { + const t0 = Date.now(); + try { + switch (variant) { + case "images": { + r = await Reflect.apply(client.imageSearch, client, [query]); + break; + } + case "news": { + r = await Reflect.apply(client.newsSearch, client, [query]); + break; + } + case "web": + default: { + r = await Reflect.apply(client.webSearch, client, [query]); + break; + } + } + const dt = Date.now() - t0; + this.logger.info( + `Search took ${dt}ms, ${client.constructor.name}(${variant})`, + { searchDt: dt, variant, client: client.constructor.name }, + ); + break outerLoop; + } catch (err) { + lastError = err; + const dt = Date.now() - t0; + this.logger.warn( + `Failed to do ${variant} search using ${client.constructor.name}`, + { err, variant, searchDt: dt }, + ); + } + } + + if (r?.length) { + const nowDate = new Date(); + const record = SERPResult.from({ + query, + queryDigest, + response: r, + createdAt: nowDate, + expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs), }); - delete crawlerOptions.html; + this.batchedCaches.push(record); + } else if (lastError) { + throw lastError; + } - const crawlOpts = await this.crawler.configure(crawlerOptions); - const searchQuery = searchExplicitOperators.addTo(q || noSlashPath); - - let fetchNum = count; - if ((page ?? 1) === 1) { - fetchNum = count > 10 ? 30 : 20; - } - - let fallbackQuery: string | undefined; - let chargeAmountScaler = 1; - if (searchEngine === 'bing') { - this.threadLocal.set('bing-preferred', true); - chargeAmountScaler = 3; - } - - if (variant !== 'web') { - chargeAmountScaler = 5; - } - - // Search with fallback logic if enabled - const searchParams = { - variant, - provider: searchEngine, - q: searchQuery, - num: fetchNum, - gl, - hl, - location, - page, - }; - - const { results, query: successQuery, tryTimes } = await this.searchWithFallback( - searchParams, fallback, crawlerOptions.noCache - ); - chargeAmountScaler *= tryTimes; - - fallbackQuery = successQuery !== searchQuery ? successQuery : undefined; - - if (!results.length) { - throw new AssertionFailureError(`No search results available for query ${searchQuery}`); - } - - if (crawlOpts.timeoutMs && crawlOpts.timeoutMs < 30_000) { - delete crawlOpts.timeoutMs; - } - - - let lastScrapped: any[] | undefined; - const targetResultCount = crawlWithoutContent ? count : count + 2; - const trimmedResults: any[] = results.filter((x) => Boolean(x.link)).slice(0, targetResultCount).map((x) => this.mapToFinalResults(x)); - trimmedResults.toString = function () { - let r = this.map((x, i) => x ? Reflect.apply(x.toString, x, [i]) : '').join('\n\n').trimEnd() + '\n'; - if (fallbackQuery) { - r = `Fallback query: ${fallbackQuery}\n\n${r}`; - } - return r; - }; - if (!crawlerOptions.respondWith.includes('no-content') && - ['html', 'text', 'shot', 'markdown', 'content'].some((x) => crawlerOptions.respondWith.includes(x)) - ) { - for (const x of trimmedResults) { - x.content ??= ''; - } - } - const assigningOfGeneralMixins = Promise.allSettled( - trimmedResults.map((x) => this.assignGeneralMixin(x)) + return r as WebSearchEntry[]; + } catch (err: any) { + if (cache) { + this.logger.warn( + `Failed to fetch search result, but a stale cache is available. falling back to stale cache`, + { err: marshalErrorLike(err) }, ); - let it; + return cache.response as any; + } - if (crawlWithoutContent || count === 0) { - it = toAsyncGenerator(trimmedResults); - await assigningOfGeneralMixins; - } else { - it = this.fetchSearchResults(crawlerOptions.respondWith, trimmedResults, crawlOpts, - CrawlerOptions.from({ ...crawlerOptions, cacheTolerance: crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs }), - count, - ); - } + throw err; + } + } - if (!ctx.accepts('text/plain') && ctx.accepts('text/event-stream')) { - const sseStream = new OutputServerEventStream(); - rpcReflect.return(sseStream); - try { - for await (const scrapped of it) { - if (!scrapped) { - continue; - } - if (rpcReflect.signal.aborted) { - break; - } + mapToFinalResults(input: WebSearchEntry) { + const whitelistedProps = [ + "imageUrl", + "imageWidth", + "imageHeight", + "source", + "date", + "siteLinks", + ]; + const result = { + title: input.title, + url: input.link, + description: Reflect.get(input, "snippet"), + ..._.pick(input, whitelistedProps), + }; - chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery); - lastScrapped = scrapped; + return result; + } - if (fallbackQuery) { - sseStream.write({ - event: 'meta', - data: { fallback: fallbackQuery }, - }); - } + async assignGeneralMixin(result: FormattedPage) { + const collectFavicon = this.threadLocal.get("collect-favicon"); - sseStream.write({ - event: 'data', - data: scrapped, - }); - } - } catch (err: any) { - this.logger.error(`Failed to collect search result for query ${searchQuery}`, - { err: marshalErrorLike(err) } - ); - sseStream.write({ - event: 'error', - data: marshalErrorLike(err), - }); - } - - sseStream.end(); - - return sseStream; - } - - let earlyReturn = false; - if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) { - let earlyReturnTimer: ReturnType | undefined; - const setEarlyReturnTimer = () => { - if (earlyReturnTimer) { - return; - } - earlyReturnTimer = setTimeout(async () => { - if (!lastScrapped) { - return; - } - await assigningOfGeneralMixins; - chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery); - - rpcReflect.return(lastScrapped); - earlyReturn = true; - }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs); - }; - - for await (const scrapped of it) { - lastScrapped = scrapped; - if (rpcReflect.signal.aborted || earlyReturn) { - break; - } - if (_.some(scrapped, (x) => this.pageQualified(x))) { - setEarlyReturnTimer(); - } - if (!this.searchResultsQualified(scrapped, count)) { - continue; - } - if (earlyReturnTimer) { - clearTimeout(earlyReturnTimer); - } - await assigningOfGeneralMixins; - chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery); - - return scrapped; - } - - if (earlyReturnTimer) { - clearTimeout(earlyReturnTimer); - } - - if (!lastScrapped) { - throw new AssertionFailureError(`No content available for query ${searchQuery}`); - } - - if (!earlyReturn) { - await assigningOfGeneralMixins; - chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery); - } - - return lastScrapped; - } - - let earlyReturnTimer: ReturnType | undefined; - const setEarlyReturnTimer = () => { - if (earlyReturnTimer) { - return; - } - earlyReturnTimer = setTimeout(async () => { - if (!lastScrapped) { - return; - } - await assigningOfGeneralMixins; - chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery); - - rpcReflect.return(assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null })); - earlyReturn = true; - }, ((crawlerOptions.timeout || 0) * 1000) || this.reasonableDelayMs); - }; - - for await (const scrapped of it) { - lastScrapped = scrapped; - if (rpcReflect.signal.aborted || earlyReturn) { - break; - } - if (_.some(scrapped, (x) => this.pageQualified(x))) { - setEarlyReturnTimer(); - } - - if (!this.searchResultsQualified(scrapped, count)) { - continue; - } - - if (earlyReturnTimer) { - clearTimeout(earlyReturnTimer); - } - await assigningOfGeneralMixins; - chargeAmount = this.assignChargeAmount(scrapped, count, chargeAmountScaler, fallbackQuery); - - return assignTransferProtocolMeta(`${scrapped}`, { contentType: 'text/plain', envelope: null }); - } - - if (earlyReturnTimer) { - clearTimeout(earlyReturnTimer); - } - - if (!lastScrapped) { - throw new AssertionFailureError(`No content available for query ${searchQuery}`); - } - - if (!earlyReturn) { - await assigningOfGeneralMixins; - chargeAmount = this.assignChargeAmount(lastScrapped, count, chargeAmountScaler, fallbackQuery); - } - - return assignTransferProtocolMeta(`${lastScrapped}`, { contentType: 'text/plain', envelope: null }); + if (collectFavicon && result.url) { + const url = new URL(result.url); + Reflect.set(result, "favicon", await this.getFavicon(url.origin)); } - /** - * Search with fallback to progressively shorter queries if no results found - * @param params Search parameters - * @param useFallback Whether to use the fallback mechanism - * @param noCache Whether to bypass cache - * @returns Search response and the successful query - */ - async searchWithFallback( - params: SerperSearchQueryParams & { variant: 'web' | 'images' | 'news'; provider?: string; }, - useFallback: boolean = false, - noCache: boolean = false - ) { - // Try original query first - const originalQuery = params.q; - const containsRTL = /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test(originalQuery); - - // Extract results based on variant - let tryTimes = 1; - const results = await this.cachedSearch(params.variant, params, noCache); - if (results.length || !useFallback) { - return { results, query: params.q, tryTimes }; - } - - let queryTerms = originalQuery.split(/\s+/); - const lastResort = containsRTL ? queryTerms.slice(queryTerms.length - 2) : queryTerms.slice(0, 2); - - this.logger.info(`No results for "${originalQuery}", trying fallback queries`); - - let terms: string[] = []; - // fallback n times - const n = 4; - - while (tryTimes < n) { - const delta = Math.ceil(queryTerms.length / n) * tryTimes; - terms = containsRTL ? queryTerms.slice(delta) : queryTerms.slice(0, queryTerms.length - delta); - const query = terms.join(' '); - if (!query) { - break; - } - tryTimes += 1; - this.logger.info(`Retrying search with fallback query: "${query}"`); - const fallbackParams = { ...params, q: query }; - const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache); - if (fallbackResults.length > 0) { - return { results: fallbackResults, query: fallbackParams.q, tryTimes }; - } - } - - if (terms.length > lastResort.length) { - const query = lastResort.join(' '); - this.logger.info(`Retrying search with fallback query: "${query}"`); - const fallbackParams = { ...params, q: query }; - tryTimes += 1; - const fallbackResults = await this.cachedSearch(params.variant, fallbackParams, noCache); - - if (fallbackResults.length > 0) { - return { results: fallbackResults, query, tryTimes }; - } - } - - return { results, query: originalQuery, tryTimes }; - } - - async *fetchSearchResults( - mode: string | 'markdown' | 'html' | 'text' | 'screenshot' | 'favicon' | 'content', - searchResults?: FormattedPage[], - options?: ExtraScrappingOptions, - crawlerOptions?: CrawlerOptions, - count?: number, - ) { - if (!searchResults) { - return; - } - const urls = searchResults.map((x) => new URL(x.url!)); - const snapshotMap = new WeakMap(); - for await (const scrapped of this.crawler.scrapMany(urls, options, crawlerOptions)) { - const mapped = scrapped.map((x, i) => { - if (!x) { - return {}; - } - if (snapshotMap.has(x)) { - return snapshotMap.get(x); - } - return this.crawler.formatSnapshotWithPDFSideLoad(mode, x, urls[i], undefined, options).then((r) => { - snapshotMap.set(x, r); - - return r; - }).catch((err) => { - this.logger.error(`Failed to format snapshot for ${urls[i].href}`, { err: marshalErrorLike(err) }); - - return {}; - }); - }); - - const resultArray = await Promise.all(mapped) as FormattedPage[]; - for (const [i, v] of resultArray.entries()) { - if (v) { - Object.assign(searchResults[i], v); - } - } - - yield this.reOrganizeSearchResults(searchResults, count); - } - } - - reOrganizeSearchResults(searchResults: FormattedPage[], count?: number) { - const targetResultCount = count || this.targetResultCount; - const [qualifiedPages, unqualifiedPages] = _.partition(searchResults, (x) => this.pageQualified(x)); - const acceptSet = new Set(qualifiedPages); - - const n = targetResultCount - qualifiedPages.length; - for (const x of unqualifiedPages.slice(0, n >= 0 ? n : 0)) { - acceptSet.add(x); - } - - const filtered = searchResults.filter((x) => acceptSet.has(x)).slice(0, targetResultCount); - - const resultArray = filtered; - - resultArray.toString = searchResults.toString; - - return resultArray; - } - - assignChargeAmount(formatted: FormattedPage[], num: number, scaler: number, fallbackQuery?: string) { - let contentCharge = 0; - for (const x of formatted) { - const itemAmount = this.crawler.assignChargeAmount(x) || 0; - - if (!itemAmount) { - continue; - } - - contentCharge += itemAmount; - } - - const numCharge = Math.ceil(formatted.length / 10) * 10000 * scaler; - - const final = Math.max(contentCharge, numCharge); - - if (final === numCharge) { - for (const x of formatted) { - x.usage = { tokens: Math.ceil(numCharge / formatted.length) }; - } - } - - const metadata: Record = { usage: { tokens: final } }; - if (fallbackQuery) { - metadata.fallback = fallbackQuery; - } - - assignMeta(formatted, metadata); - - return final; - } - - pageQualified(formattedPage: FormattedPage) { - return formattedPage.title && - formattedPage.content || - formattedPage.screenshotUrl || - formattedPage.pageshotUrl || - formattedPage.text || - formattedPage.html; - } - - searchResultsQualified(results: FormattedPage[], targetResultCount = this.targetResultCount) { - return _.every(results, (x) => this.pageQualified(x)) && results.length >= targetResultCount; - } - - async getFavicon(domain: string) { - const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`; - - try { - const response = await fetch(url); - if (!response.ok) { - return ''; - } - const ab = await response.arrayBuffer(); - const buffer = Buffer.from(ab); - const base64 = buffer.toString('base64'); - return `data:image/png;base64,${base64}`; - } catch (error: any) { - this.logger.warn(`Failed to get favicon base64 string`, { err: marshalErrorLike(error) }); - return ''; - } - } - - *iterProviders(preference?: string, _variant?: string) { - const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY; - const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY); - - if (preference === 'bing') { - if (preferScraping) { - yield this.standaloneFallback; - } - if (includeSerper) { - yield this.serperBing; - yield this.serperGoogle; - } - yield this.googleSerp; - if (includeSerper) { - yield this.serperGoogle; - } - - return; - } - - if (preference === 'google') { - if (preferScraping) { - yield this.standaloneFallback; - yield this.googleSerp; - } - yield this.googleSerp; - if (includeSerper) { - yield this.serperGoogle; - yield this.serperGoogle; - } - - return; - } - - if (preferScraping) { - yield this.standaloneFallback; - yield this.googleSerp; - } - yield this.googleSerp; - if (includeSerper) { - yield this.serperGoogle; - yield this.serperGoogle; - yield this.serperBing; - } - } - - async cachedSearch(variant: 'web' | 'news' | 'images', query: Record, noCache?: boolean): Promise { - const queryDigest = objHashMd5B64Of({ ...query, variant }); - const provider = query.provider; - Reflect.deleteProperty(query, 'provider'); - let cache; - if (!noCache) { - cache = (await SERPResult.fromFirestoreQuery( - SERPResult.COLLECTION.where('queryDigest', '==', queryDigest) - .orderBy('createdAt', 'desc') - .limit(1) - ))[0]; - if (cache) { - const age = Date.now() - cache.createdAt.valueOf(); - const stale = cache.createdAt.valueOf() < (Date.now() - this.cacheValidMs); - this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`, { - query, digest: queryDigest, age, stale - }); - - if (!stale) { - return cache.response as any; - } - } - } - - try { - let r: any[] | undefined; - let lastError; - outerLoop: - for (const client of this.iterProviders(provider, variant)) { - const t0 = Date.now(); - try { - switch (variant) { - case 'images': { - r = await Reflect.apply(client.imageSearch, client, [query]); - break; - } - case 'news': { - r = await Reflect.apply(client.newsSearch, client, [query]); - break; - } - case 'web': - default: { - r = await Reflect.apply(client.webSearch, client, [query]); - break; - } - } - const dt = Date.now() - t0; - this.logger.info(`Search took ${dt}ms, ${client.constructor.name}(${variant})`, { searchDt: dt, variant, client: client.constructor.name }); - break outerLoop; - } catch (err) { - lastError = err; - const dt = Date.now() - t0; - this.logger.warn(`Failed to do ${variant} search using ${client.constructor.name}`, { err, variant, searchDt: dt, }); - } - } - - if (r?.length) { - const nowDate = new Date(); - const record = SERPResult.from({ - query, - queryDigest, - response: r, - createdAt: nowDate, - expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs) - }); - - this.batchedCaches.push(record); - } else if (lastError) { - throw lastError; - } - - return r as WebSearchEntry[]; - } catch (err: any) { - if (cache) { - this.logger.warn(`Failed to fetch search result, but a stale cache is available. falling back to stale cache`, { err: marshalErrorLike(err) }); - - return cache.response as any; - } - - throw err; - } - } - - mapToFinalResults(input: WebSearchEntry) { - const whitelistedProps = [ - 'imageUrl', 'imageWidth', 'imageHeight', 'source', 'date', 'siteLinks' - ]; - const result = { - title: input.title, - url: input.link, - description: Reflect.get(input, 'snippet'), - ..._.pick(input, whitelistedProps), - }; - - return result; - } - - async assignGeneralMixin(result: FormattedPage) { - const collectFavicon = this.threadLocal.get('collect-favicon'); - - if (collectFavicon && result.url) { - const url = new URL(result.url); - Reflect.set(result, 'favicon', await this.getFavicon(url.origin)); - } - - Object.setPrototypeOf(result, searchResultProto); - } + Object.setPrototypeOf(result, searchResultProto); + } } const dataItems = [ - { key: 'title', label: 'Title' }, - { key: 'source', label: 'Source' }, - { key: 'url', label: 'URL Source' }, - { key: 'imageUrl', label: 'Image URL' }, - { key: 'description', label: 'Description' }, - { key: 'publishedTime', label: 'Published Time' }, - { key: 'imageWidth', label: 'Image Width' }, - { key: 'imageHeight', label: 'Image Height' }, - { key: 'date', label: 'Date' }, - { key: 'favicon', label: 'Favicon' }, + { key: "title", label: "Title" }, + { key: "source", label: "Source" }, + { key: "url", label: "URL Source" }, + { key: "imageUrl", label: "Image URL" }, + { key: "description", label: "Description" }, + { key: "publishedTime", label: "Published Time" }, + { key: "imageWidth", label: "Image Width" }, + { key: "imageHeight", label: "Image Height" }, + { key: "date", label: "Date" }, + { key: "favicon", label: "Favicon" }, ]; const searchResultProto = { - toString(this: FormattedPage, i?: number) { - const chunks = []; - for (const item of dataItems) { - const v = Reflect.get(this, item.key); - if (typeof v !== 'undefined') { - if (i === undefined) { - chunks.push(`[${item.label}]: ${v}`); - } else { - chunks.push(`[${i + 1}] ${item.label}: ${v}`); - } - } + toString(this: FormattedPage, i?: number) { + const chunks = []; + for (const item of dataItems) { + const v = Reflect.get(this, item.key); + if (typeof v !== "undefined") { + if (i === undefined) { + chunks.push(`[${item.label}]: ${v}`); + } else { + chunks.push(`[${i + 1}] ${item.label}: ${v}`); } - - if (this.content) { - chunks.push(`\n${this.content}`); - } - - if (this.images) { - const imageSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Images:`]; - for (const [k, v] of Object.entries(this.images)) { - imageSummaryChunks.push(`- ![${k}](${v})`); - } - if (imageSummaryChunks.length === 1) { - imageSummaryChunks.push('This page does not seem to contain any images.'); - } - chunks.push(imageSummaryChunks.join('\n')); - } - if (this.links) { - const linkSummaryChunks = [`${i === undefined ? '' : `[${i + 1}] `}Links/Buttons:`]; - if (Array.isArray(this.links)) { - for (const [k, v] of this.links) { - linkSummaryChunks.push(`- [${k}](${v})`); - } - } else { - for (const [k, v] of Object.entries(this.links)) { - linkSummaryChunks.push(`- [${k}](${v})`); - } - } - if (linkSummaryChunks.length === 1) { - linkSummaryChunks.push('This page does not seem to contain any buttons/links.'); - } - chunks.push(linkSummaryChunks.join('\n')); - } - - return chunks.join('\n'); + } } + + if (this.content) { + chunks.push(`\n${this.content}`); + } + + if (this.images) { + const imageSummaryChunks = [ + `${i === undefined ? "" : `[${i + 1}] `}Images:`, + ]; + for (const [k, v] of Object.entries(this.images)) { + imageSummaryChunks.push(`- ![${k}](${v})`); + } + if (imageSummaryChunks.length === 1) { + imageSummaryChunks.push( + "This page does not seem to contain any images.", + ); + } + chunks.push(imageSummaryChunks.join("\n")); + } + if (this.links) { + const linkSummaryChunks = [ + `${i === undefined ? "" : `[${i + 1}] `}Links/Buttons:`, + ]; + if (Array.isArray(this.links)) { + for (const [k, v] of this.links) { + linkSummaryChunks.push(`- [${k}](${v})`); + } + } else { + for (const [k, v] of Object.entries(this.links)) { + linkSummaryChunks.push(`- [${k}](${v})`); + } + } + if (linkSummaryChunks.length === 1) { + linkSummaryChunks.push( + "This page does not seem to contain any buttons/links.", + ); + } + chunks.push(linkSummaryChunks.join("\n")); + } + + return chunks.join("\n"); + }, }; diff --git a/src/api/serp.ts b/src/api/serp.ts index d718d82..557e2cc 100644 --- a/src/api/serp.ts +++ b/src/api/serp.ts @@ -1,593 +1,713 @@ -import { singleton } from 'tsyringe'; +import { singleton } from "tsyringe"; import { - RPCHost, RPCReflection, assignMeta, RawString, - ParamValidationError, - assignTransferProtocolMeta, -} from 'civkit/civ-rpc'; -import { marshalErrorLike } from 'civkit/lang'; -import _ from 'lodash'; + RPCHost, + RPCReflection, + assignMeta, + RawString, + ParamValidationError, + assignTransferProtocolMeta, +} from "civkit/civ-rpc"; +import { marshalErrorLike } from "civkit/lang"; +import _ from "lodash"; -import { RateLimitControl, RateLimitDesc, RateLimitTriggeredError } from '../shared/services/rate-limit'; +import { + RateLimitControl, + RateLimitDesc, + RateLimitTriggeredError, +} from "../shared/services/rate-limit"; -import { GlobalLogger } from '../services/logger'; -import { AsyncLocalContext } from '../services/async-context'; -import { Context, Ctx, Method, Param, RPCReflect } from '../services/registry'; -import { OutputServerEventStream } from '../lib/transform-server-event-stream'; -import { ApiTokenAccount, AuthDTO } from '../dto/auth'; -import { WORLD_LANGUAGES } from '../shared/3rd-party/serper-search'; -import { GoogleSERP } from '../services/serp/google'; -import { WebSearchEntry } from '../services/serp/compat'; -import { CrawlerOptions } from '../dto/crawler-options'; -import { ScrappingOptions } from '../services/serp/puppeteer'; -import { objHashMd5B64Of } from 'civkit/hash'; -import { SERPResult } from '../db/searched'; -import { SerperBingSearchService, SerperGoogleSearchService } from '../services/serp/serper'; -import { LRUCache } from 'lru-cache'; -import { API_CALL_STATUS } from '../shared/db/api-roll'; -import { SecretExposer } from '../shared/services/secrets'; -import { StandaloneSearchFallbackService } from '../services/serp/standalone-fallback'; -import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; +import { GlobalLogger } from "../services/logger"; +import { AsyncLocalContext } from "../services/async-context"; +import { Context, Ctx, Method, Param, RPCReflect } from "../services/registry"; +import { OutputServerEventStream } from "../lib/transform-server-event-stream"; +import { ApiTokenAccount, AuthDTO } from "../dto/auth"; +import { WORLD_LANGUAGES } from "../shared/3rd-party/serper-search"; +import { GoogleSERP } from "../services/serp/google"; +import { WebSearchEntry } from "../services/serp/compat"; +import { CrawlerOptions } from "../dto/crawler-options"; +import { ScrappingOptions } from "../services/serp/puppeteer"; +import { objHashMd5B64Of } from "civkit/hash"; +import { SERPResult } from "../db/searched"; +import { + SerperBingSearchService, + SerperGoogleSearchService, +} from "../services/serp/serper"; +import { LRUCache } from "lru-cache"; +import { API_CALL_STATUS } from "../shared/db/api-roll"; +import { SecretExposer } from "../shared/services/secrets"; +import { StandaloneSearchFallbackService } from "../services/serp/standalone-fallback"; +import { STANDALONE_BOOT_TIMEOUT_MS } from "../services/boot-timeouts"; type RateLimitCache = { - blockedUntil?: Date; - user?: ApiTokenAccount; + blockedUntil?: Date; + user?: ApiTokenAccount; }; const indexProto = { - toString: function (): string { - return _(this) - .toPairs() - .map(([k, v]) => k ? `[${_.upperFirst(_.lowerCase(k))}] ${v}` : '') - .value() - .join('\n') + '\n'; - } + toString: function (): string { + return ( + _(this) + .toPairs() + .map(([k, v]) => (k ? `[${_.upperFirst(_.lowerCase(k))}] ${v}` : "")) + .value() + .join("\n") + "\n" + ); + }, }; +function derivePublicOrigins(ctx: Context) { + const homepage = "https://github.com/xixu-me/xread"; + const currentUrl = new URL(ctx.origin); + let crawlOrigin = currentUrl.origin; + let searchOrigin = currentUrl.origin; + + if (ctx.hostname.startsWith("r.")) { + const derivedSearchUrl = new URL(currentUrl.origin); + derivedSearchUrl.hostname = `s.${ctx.hostname.slice(2)}`; + searchOrigin = derivedSearchUrl.origin; + } else if (ctx.hostname.startsWith("s.")) { + const derivedCrawlUrl = new URL(currentUrl.origin); + derivedCrawlUrl.hostname = `r.${ctx.hostname.slice(2)}`; + crawlOrigin = derivedCrawlUrl.origin; + } + + return { crawlOrigin, searchOrigin, homepage }; +} + @singleton() export class SerpHost extends RPCHost { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - cacheRetentionMs = 1000 * 3600 * 24 * 7; - cacheValidMs = 1000 * 3600; - pageCacheToleranceMs = 1000 * 3600 * 24; + cacheRetentionMs = 1000 * 3600 * 24 * 7; + cacheValidMs = 1000 * 3600; + pageCacheToleranceMs = 1000 * 3600 * 24; - reasonableDelayMs = 15_000; + reasonableDelayMs = 15_000; - targetResultCount = 5; + targetResultCount = 5; - highFreqKeyCache = new LRUCache({ - max: 256, - ttl: 60 * 60 * 1000, - updateAgeOnGet: false, - updateAgeOnHas: false, + highFreqKeyCache = new LRUCache({ + max: 256, + ttl: 60 * 60 * 1000, + updateAgeOnGet: false, + updateAgeOnHas: false, + }); + + batchedCaches: SERPResult[] = []; + + async getIndex(ctx: Context, auth?: AuthDTO) { + const { searchOrigin, homepage } = derivePublicOrigins(ctx); + const indexObject: Record = + Object.create(indexProto); + Object.assign(indexObject, { + textSearch: `${searchOrigin}/search?q=YOUR_SEARCH_QUERY`, + jsonSearch: `${searchOrigin}/?q=YOUR_SEARCH_QUERY`, + repository: homepage, }); - batchedCaches: SERPResult[] = []; + if (auth && auth.user) { + indexObject[""] = undefined; + indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`; + indexObject.balanceLeft = auth.user.wallet.total_balance; + } - async getIndex(ctx: Context, auth?: AuthDTO) { - const indexObject: Record = Object.create(indexProto); - Object.assign(indexObject, { - usage1: 'https://r.example.com/YOUR_URL', - usage2: 'https://s.example.com/YOUR_SEARCH_QUERY', - usage3: `${ctx.origin}/?q=YOUR_SEARCH_QUERY`, - homepage: 'https://example.com/xread', + return indexObject; + } + + constructor( + protected globalLogger: GlobalLogger, + protected rateLimitControl: RateLimitControl, + protected threadLocal: AsyncLocalContext, + protected secretExposer: SecretExposer, + protected googleSerp: GoogleSERP, + protected standaloneFallback: StandaloneSearchFallbackService, + protected serperGoogle: SerperGoogleSearchService, + protected serperBing: SerperBingSearchService, + ) { + super(...arguments); + + setInterval( + () => { + const thisBatch = this.batchedCaches; + this.batchedCaches = []; + if (!thisBatch.length) { + return; + } + const batch = SERPResult.DB.batch(); + + for (const x of thisBatch) { + batch.set(SERPResult.COLLECTION.doc(), x.degradeForFireStore()); + } + batch + .commit() + .then(() => { + this.logger.debug(`Saved ${thisBatch.length} caches by batch`); + }) + .catch((err) => { + this.logger.warn(`Failed to cache search result in batch`, { err }); + }); + }, + 1000 * 10 + Math.round(1000 * Math.random()), + ).unref(); + } + + override async init() { + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + + this.emit("ready"); + } + + override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { + return super.dependencyReady(timeoutMilliseconds); + } + + @Method({ + name: "searchIndex", + ext: { + http: { + action: ["get", "post"], + path: "/", + }, + }, + tags: ["search"], + returnType: [String, OutputServerEventStream, RawString], + }) + @Method({ + ext: { + http: { + action: ["get", "post"], + }, + }, + tags: ["search"], + returnType: [String, OutputServerEventStream, RawString], + }) + async search( + @RPCReflect() rpcReflect: RPCReflection, + @Ctx() ctx: Context, + crawlerOptions: CrawlerOptions, + auth: AuthDTO, + @Param("type", { type: new Set(["web", "images", "news"]), default: "web" }) + variant: "web" | "images" | "news", + @Param("q") q?: string, + @Param("provider", { type: new Set(["google", "bing"]) }) + searchEngine?: "google" | "bing", + @Param("num", { validate: (v: number) => v >= 0 && v <= 20 }) + num?: number, + @Param("gl", { validate: (v: string) => /^[a-z]{2}$/i.test(v || "") }) + gl?: string, + @Param("hl", { + validate: (v: string) => + WORLD_LANGUAGES.some((l) => l.code === v) || + /^[a-z]{2}$/i.test(v || ""), + }) + _hl?: string, + @Param("location") location?: string, + @Param("page") page?: number, + @Param("fallback") fallback?: boolean, + ) { + const authToken = auth.bearerToken; + let highFreqKey: RateLimitCache | undefined; + if (authToken && this.highFreqKeyCache.has(authToken)) { + highFreqKey = this.highFreqKeyCache.get(authToken)!; + auth.user = highFreqKey.user; + auth.uid = highFreqKey.user?.user_id; + } + + const uid = await auth.solveUID(); + if (!q) { + if (ctx.path === "/") { + const indexObject = await this.getIndex(ctx, auth); + if ( + !ctx.accepts("text/plain") && + (ctx.accepts("text/json") || ctx.accepts("application/json")) + ) { + return indexObject; + } + + return assignTransferProtocolMeta(`${indexObject}`, { + contentType: "text/plain; charset=utf-8", + envelope: null, }); + } + throw new ParamValidationError({ + path: "q", + message: `Required but not provided`, + }); + } + const user = await auth.assertUser(); - if (auth && auth.user) { - indexObject[''] = undefined; - indexObject.authenticatedAs = `${auth.user.user_id} (${auth.user.full_name})`; - indexObject.balanceLeft = auth.user.wallet.total_balance; - } - - return indexObject; + if (highFreqKey?.blockedUntil) { + const now = new Date(); + const blockedTimeRemaining = + highFreqKey.blockedUntil.valueOf() - now.valueOf(); + if (blockedTimeRemaining > 0) { + this.logger.warn( + `Rate limit triggered for ${uid}, this request should have been blocked`, + ); + // throw RateLimitTriggeredError.from({ + // message: `Per UID rate limit exceeded (async)`, + // retryAfter: Math.ceil(blockedTimeRemaining / 1000), + // }); + } } - constructor( - protected globalLogger: GlobalLogger, - protected rateLimitControl: RateLimitControl, - protected threadLocal: AsyncLocalContext, - protected secretExposer: SecretExposer, - protected googleSerp: GoogleSERP, - protected standaloneFallback: StandaloneSearchFallbackService, - protected serperGoogle: SerperGoogleSearchService, - protected serperBing: SerperBingSearchService, - ) { - super(...arguments); + const PREMIUM_KEY_LIMIT = 400; + const rateLimitPolicy = auth.getRateLimits("SEARCH") || [ + parseInt(user.metadata?.speed_level) >= 2 + ? RateLimitDesc.from({ + occurrence: PREMIUM_KEY_LIMIT, + periodSeconds: 60, + }) + : RateLimitDesc.from({ + occurrence: 40, + periodSeconds: 60, + }), + ]; - setInterval(() => { - const thisBatch = this.batchedCaches; - this.batchedCaches = []; - if (!thisBatch.length) { - return; + const apiRollPromise = uid + ? this.rateLimitControl.simpleRPCUidBasedLimit( + rpcReflect, + uid, + ["SEARCH"], + ...rateLimitPolicy, + ) + : this.rateLimitControl.simpleRpcIPBasedLimit( + rpcReflect, + ctx.ip || "anonymous", + ["SEARCH"], + [[new Date(Date.now() - 60 * 1000), 20]], + ); + + if (uid && !highFreqKey) { + // Normal path + await apiRollPromise; + + if ( + rateLimitPolicy.some((x) => { + const rpm = x.occurrence / (x.periodSeconds / 60); + if (rpm >= PREMIUM_KEY_LIMIT) { + return true; + } + + return false; + }) + ) { + this.highFreqKeyCache.set(auth.bearerToken!, { + user, + }); + } + } else if (uid && highFreqKey) { + // High freq key path + apiRollPromise + .then( + // Rate limit not triggered, make sure not blocking. + () => { + delete highFreqKey.blockedUntil; + }, + // Rate limit triggered + (err) => { + if (!(err instanceof RateLimitTriggeredError)) { + return; } - const batch = SERPResult.DB.batch(); - - for (const x of thisBatch) { - batch.set(SERPResult.COLLECTION.doc(), x.degradeForFireStore()); + const now = Date.now(); + let tgtDate; + if (err.retryAfterDate) { + tgtDate = err.retryAfterDate; + } else if (err.retryAfter) { + tgtDate = new Date(now + err.retryAfter * 1000); } - batch.commit() - .then(() => { - this.logger.debug(`Saved ${thisBatch.length} caches by batch`); - }) - .catch((err) => { - this.logger.warn(`Failed to cache search result in batch`, { err }); - }); - }, 1000 * 10 + Math.round(1000 * Math.random())).unref(); - } - override async init() { - await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); - - this.emit('ready'); - } - - override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { - return super.dependencyReady(timeoutMilliseconds); - } - - @Method({ - name: 'searchIndex', - ext: { - http: { - action: ['get', 'post'], - path: '/' - } - }, - tags: ['search'], - returnType: [String, OutputServerEventStream, RawString], - }) - @Method({ - ext: { - http: { - action: ['get', 'post'], - } - }, - tags: ['search'], - returnType: [String, OutputServerEventStream, RawString], - }) - async search( - @RPCReflect() rpcReflect: RPCReflection, - @Ctx() ctx: Context, - crawlerOptions: CrawlerOptions, - auth: AuthDTO, - @Param('type', { type: new Set(['web', 'images', 'news']), default: 'web' }) - variant: 'web' | 'images' | 'news', - @Param('q') q?: string, - @Param('provider', { type: new Set(['google', 'bing']) }) - searchEngine?: 'google' | 'bing', - @Param('num', { validate: (v: number) => v >= 0 && v <= 20 }) - num?: number, - @Param('gl', { validate: (v: string) => /^[a-z]{2}$/i.test(v || '') }) gl?: string, - @Param('hl', { validate: (v: string) => WORLD_LANGUAGES.some(l => l.code === v) || /^[a-z]{2}$/i.test(v || '') }) _hl?: string, - @Param('location') location?: string, - @Param('page') page?: number, - @Param('fallback') fallback?: boolean, - ) { - const authToken = auth.bearerToken; - let highFreqKey: RateLimitCache | undefined; - if (authToken && this.highFreqKeyCache.has(authToken)) { - highFreqKey = this.highFreqKeyCache.get(authToken)!; - auth.user = highFreqKey.user; - auth.uid = highFreqKey.user?.user_id; - } - - const uid = await auth.solveUID(); - if (!q) { - if (ctx.path === '/') { - const indexObject = await this.getIndex(ctx, auth); - if (!ctx.accepts('text/plain') && (ctx.accepts('text/json') || ctx.accepts('application/json'))) { - return indexObject; + if (tgtDate) { + const dt = tgtDate.valueOf() - now; + highFreqKey.blockedUntil = tgtDate; + setTimeout(() => { + if (highFreqKey.blockedUntil === tgtDate) { + delete highFreqKey.blockedUntil; } - - return assignTransferProtocolMeta(`${indexObject}`, - { contentType: 'text/plain; charset=utf-8', envelope: null } - ); + }, dt).unref(); } - throw new ParamValidationError({ - path: 'q', - message: `Required but not provided` + }, + ) + .finally(async () => { + // Always asynchronously update user(wallet); + const user = await auth.getBrief().catch(() => undefined); + if (user) { + highFreqKey.user = user; + } + }); + } + + let chargeAmount = 0; + rpcReflect.finally(async () => { + if (chargeAmount) { + if (uid) { + auth.reportUsage(chargeAmount, `xread-search`).catch((err) => { + this.logger.warn(`Unable to report usage for ${uid}`, { + err: marshalErrorLike(err), + }); + }); + } + try { + const apiRoll = await apiRollPromise; + apiRoll.chargeAmount = chargeAmount; + await apiRoll.save({ merge: true }); + } catch (_err) { + await this.rateLimitControl + .record({ + uid, + tags: [rpcReflect.name.toUpperCase()], + status: API_CALL_STATUS.SUCCESS, + chargeAmount, + }) + .save() + .catch((err) => { + this.logger.warn(`Failed to save rate limit record`, { + err: marshalErrorLike(err), + }); }); } - const user = await auth.assertUser(); + } + }); - if (highFreqKey?.blockedUntil) { - const now = new Date(); - const blockedTimeRemaining = (highFreqKey.blockedUntil.valueOf() - now.valueOf()); - if (blockedTimeRemaining > 0) { - this.logger.warn(`Rate limit triggered for ${uid}, this request should have been blocked`); - // throw RateLimitTriggeredError.from({ - // message: `Per UID rate limit exceeded (async)`, - // retryAfter: Math.ceil(blockedTimeRemaining / 1000), - // }); - } + let chargeAmountScaler = 1; + if (searchEngine === "bing") { + chargeAmountScaler = 3; + } + if (variant !== "web") { + chargeAmountScaler = 5; + } + + let realQuery = q; + let queryTerms = q.split(/\s+/g).filter((x) => !!x); + + let results = await this.cachedSearch( + variant, + { + provider: searchEngine, + q, + num, + gl, + // hl, + location, + page, + }, + crawlerOptions, + ); + + if (fallback && !results?.length && (!page || page === 1)) { + let tryTimes = 1; + const containsRTL = + /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test( + q, + ); + const lastResort = ( + containsRTL + ? queryTerms.slice(queryTerms.length - 2) + : queryTerms.slice(0, 2) + ).join(" "); + const n = 4; + let terms: string[] = []; + while (tryTimes < n) { + const delta = Math.ceil(queryTerms.length / n) * tryTimes; + terms = containsRTL + ? queryTerms.slice(delta) + : queryTerms.slice(0, queryTerms.length - delta); + const query = terms.join(" "); + if (!query) { + break; } - - const PREMIUM_KEY_LIMIT = 400; - const rateLimitPolicy = auth.getRateLimits('SEARCH') || [ - parseInt(user.metadata?.speed_level) >= 2 ? - RateLimitDesc.from({ - occurrence: PREMIUM_KEY_LIMIT, - periodSeconds: 60 - }) : - RateLimitDesc.from({ - occurrence: 40, - periodSeconds: 60 - }) - ]; - - const apiRollPromise = uid ? - this.rateLimitControl.simpleRPCUidBasedLimit( - rpcReflect, uid, ['SEARCH'], - ...rateLimitPolicy - ) : - this.rateLimitControl.simpleRpcIPBasedLimit( - rpcReflect, ctx.ip || 'anonymous', ['SEARCH'], - [[new Date(Date.now() - 60 * 1000), 20]] - ); - - if (uid && !highFreqKey) { - // Normal path - await apiRollPromise; - - if (rateLimitPolicy.some( - (x) => { - const rpm = x.occurrence / (x.periodSeconds / 60); - if (rpm >= PREMIUM_KEY_LIMIT) { - return true; - } - - return false; - }) - ) { - this.highFreqKeyCache.set(auth.bearerToken!, { - user, - }); - } - } else if (uid && highFreqKey) { - // High freq key path - apiRollPromise.then( - // Rate limit not triggered, make sure not blocking. - () => { - delete highFreqKey.blockedUntil; - }, - // Rate limit triggered - (err) => { - if (!(err instanceof RateLimitTriggeredError)) { - return; - } - const now = Date.now(); - let tgtDate; - if (err.retryAfterDate) { - tgtDate = err.retryAfterDate; - } else if (err.retryAfter) { - tgtDate = new Date(now + err.retryAfter * 1000); - } - - if (tgtDate) { - const dt = tgtDate.valueOf() - now; - highFreqKey.blockedUntil = tgtDate; - setTimeout(() => { - if (highFreqKey.blockedUntil === tgtDate) { - delete highFreqKey.blockedUntil; - } - }, dt).unref(); - } - } - ).finally(async () => { - // Always asynchronously update user(wallet); - const user = await auth.getBrief().catch(() => undefined); - if (user) { - highFreqKey.user = user; - } - }); + if (realQuery === query) { + continue; } - - let chargeAmount = 0; - rpcReflect.finally(async () => { - if (chargeAmount) { - if (uid) { - auth.reportUsage(chargeAmount, `xread-search`).catch((err) => { - this.logger.warn(`Unable to report usage for ${uid}`, { err: marshalErrorLike(err) }); - }); - } - try { - const apiRoll = await apiRollPromise; - apiRoll.chargeAmount = chargeAmount; - await apiRoll.save({ merge: true }); - } catch (_err) { - await this.rateLimitControl.record({ - uid, - tags: [rpcReflect.name.toUpperCase()], - status: API_CALL_STATUS.SUCCESS, - chargeAmount, - }).save().catch((err) => { - this.logger.warn(`Failed to save rate limit record`, { err: marshalErrorLike(err) }); - }); - } - } - }); - - let chargeAmountScaler = 1; - if (searchEngine === 'bing') { - chargeAmountScaler = 3; - } - if (variant !== 'web') { - chargeAmountScaler = 5; - } - - let realQuery = q; - let queryTerms = q.split(/\s+/g).filter((x) => !!x); - - let results = await this.cachedSearch(variant, { + tryTimes += 1; + realQuery = query; + this.logger.info(`Retrying search with fallback query: "${realQuery}"`); + results = await this.cachedSearch( + variant, + { provider: searchEngine, - q, + q: realQuery, num, gl, // hl, location, - page, - }, crawlerOptions); - - - if (fallback && !results?.length && (!page || page === 1)) { - let tryTimes = 1; - const containsRTL = /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test(q); - const lastResort = (containsRTL ? queryTerms.slice(queryTerms.length - 2) : queryTerms.slice(0, 2)).join(' '); - const n = 4; - let terms: string[] = []; - while (tryTimes < n) { - const delta = Math.ceil(queryTerms.length / n) * tryTimes; - terms = containsRTL ? queryTerms.slice(delta) : queryTerms.slice(0, queryTerms.length - delta); - const query = terms.join(' '); - if (!query) { - break; - } - if (realQuery === query) { - continue; - } - tryTimes += 1; - realQuery = query; - this.logger.info(`Retrying search with fallback query: "${realQuery}"`); - results = await this.cachedSearch(variant, { - provider: searchEngine, - q: realQuery, - num, - gl, - // hl, - location, - }, crawlerOptions); - if (results?.length) { - break; - } - } - - if (!results?.length && realQuery.length > lastResort.length) { - realQuery = lastResort; - this.logger.info(`Retrying search with fallback query: "${realQuery}"`); - tryTimes += 1; - results = await this.cachedSearch(variant, { - provider: searchEngine, - q: realQuery, - num, - gl, - // hl, - location, - }, crawlerOptions); - } - - chargeAmountScaler *= tryTimes; + }, + crawlerOptions, + ); + if (results?.length) { + break; } + } - if (!results?.length) { - results = []; - } + if (!results?.length && realQuery.length > lastResort.length) { + realQuery = lastResort; + this.logger.info(`Retrying search with fallback query: "${realQuery}"`); + tryTimes += 1; + results = await this.cachedSearch( + variant, + { + provider: searchEngine, + q: realQuery, + num, + gl, + // hl, + location, + }, + crawlerOptions, + ); + } - const finalResults = results.map((x: any) => this.mapToFinalResults(x)); - - await Promise.all(finalResults.map((x: any) => this.assignGeneralMixin(x))); - - chargeAmount = this.assignChargeAmount(finalResults, chargeAmountScaler); - assignMeta(finalResults, { - query: realQuery, - fallback: realQuery === q ? undefined : realQuery, - }); - - return finalResults; + chargeAmountScaler *= tryTimes; } - - assignChargeAmount(items: unknown[], scaler: number) { - const numCharge = Math.ceil(items.length / 10) * 10000 * scaler; - assignMeta(items, { usage: { tokens: numCharge } }); - - return numCharge; + if (!results?.length) { + results = []; } - async getFavicon(domain: string) { - const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`; + const finalResults = results.map((x: any) => this.mapToFinalResults(x)); - try { - const response = await fetch(url); - if (!response.ok) { - return ''; - } - const ab = await response.arrayBuffer(); - const buffer = Buffer.from(ab); - const base64 = buffer.toString('base64'); - return `data:image/png;base64,${base64}`; - } catch (error: any) { - this.logger.warn(`Failed to get favicon base64 string`, { err: marshalErrorLike(error) }); - return ''; - } + await Promise.all(finalResults.map((x: any) => this.assignGeneralMixin(x))); + + chargeAmount = this.assignChargeAmount(finalResults, chargeAmountScaler); + assignMeta(finalResults, { + query: realQuery, + fallback: realQuery === q ? undefined : realQuery, + }); + + return finalResults; + } + + assignChargeAmount(items: unknown[], scaler: number) { + const numCharge = Math.ceil(items.length / 10) * 10000 * scaler; + assignMeta(items, { usage: { tokens: numCharge } }); + + return numCharge; + } + + async getFavicon(domain: string) { + const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`; + + try { + const response = await fetch(url); + if (!response.ok) { + return ""; + } + const ab = await response.arrayBuffer(); + const buffer = Buffer.from(ab); + const base64 = buffer.toString("base64"); + return `data:image/png;base64,${base64}`; + } catch (error: any) { + this.logger.warn(`Failed to get favicon base64 string`, { + err: marshalErrorLike(error), + }); + return ""; + } + } + + async configure(opts: CrawlerOptions) { + const crawlOpts: ScrappingOptions = { + proxyUrl: opts.proxyUrl, + cookies: opts.setCookies, + overrideUserAgent: opts.userAgent, + timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined, + locale: opts.locale, + referer: opts.referer, + viewport: opts.viewport, + proxyResources: opts.proxyUrl || opts.proxy?.endsWith("+") ? true : false, + allocProxy: opts.proxy?.endsWith("+") + ? opts.proxy.slice(0, -1) + : opts.proxy, + }; + + if (opts.locale) { + crawlOpts.extraHeaders ??= {}; + crawlOpts.extraHeaders["Accept-Language"] = opts.locale; } - async configure(opts: CrawlerOptions) { - const crawlOpts: ScrappingOptions = { - proxyUrl: opts.proxyUrl, - cookies: opts.setCookies, - overrideUserAgent: opts.userAgent, - timeoutMs: opts.timeout ? opts.timeout * 1000 : undefined, - locale: opts.locale, - referer: opts.referer, - viewport: opts.viewport, - proxyResources: (opts.proxyUrl || opts.proxy?.endsWith('+')) ? true : false, - allocProxy: opts.proxy?.endsWith('+') ? opts.proxy.slice(0, -1) : opts.proxy, - }; + return crawlOpts; + } - if (opts.locale) { - crawlOpts.extraHeaders ??= {}; - crawlOpts.extraHeaders['Accept-Language'] = opts.locale; - } + mapToFinalResults(input: WebSearchEntry) { + const whitelistedProps = [ + "imageUrl", + "imageWidth", + "imageHeight", + "source", + "date", + "siteLinks", + ]; + const result = { + title: input.title, + url: input.link, + description: Reflect.get(input, "snippet"), + ..._.pick(input, whitelistedProps), + }; - return crawlOpts; + return result; + } + + *iterProviders(preference?: string, _variant?: string) { + const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY; + const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY); + + if (preference === "bing") { + if (preferScraping) { + yield this.standaloneFallback; + } + if (includeSerper) { + yield this.serperBing; + yield this.serperGoogle; + } + yield this.googleSerp; + + return; } - mapToFinalResults(input: WebSearchEntry) { - const whitelistedProps = [ - 'imageUrl', 'imageWidth', 'imageHeight', 'source', 'date', 'siteLinks' - ]; - const result = { - title: input.title, - url: input.link, - description: Reflect.get(input, 'snippet'), - ..._.pick(input, whitelistedProps), - }; - - return result; - } - - *iterProviders(preference?: string, _variant?: string) { - const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY; - const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY); - - if (preference === 'bing') { - if (preferScraping) { - yield this.standaloneFallback; - } - if (includeSerper) { - yield this.serperBing; - yield this.serperGoogle; - } - yield this.googleSerp; - - return; - } - - if (preference === 'google') { - if (preferScraping) { - yield this.standaloneFallback; - yield this.googleSerp; - } - yield this.googleSerp; - yield this.googleSerp; - if (includeSerper) { - yield this.serperGoogle; - } - - return; - } - - if (preferScraping) { - yield this.standaloneFallback; - yield this.googleSerp; - } + if (preference === "google") { + if (preferScraping) { + yield this.standaloneFallback; yield this.googleSerp; - if (includeSerper) { - yield this.serperGoogle; - yield this.serperGoogle; - } + } + yield this.googleSerp; + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + } + + return; } - async cachedSearch(variant: 'web' | 'news' | 'images', query: Record, opts: CrawlerOptions) { - const queryDigest = objHashMd5B64Of({ ...query, variant }); - const provider = query.provider; - Reflect.deleteProperty(query, 'provider'); - const noCache = opts.noCache; - let cache; - if (!noCache) { - cache = (await SERPResult.fromFirestoreQuery( - SERPResult.COLLECTION.where('queryDigest', '==', queryDigest) - .orderBy('createdAt', 'desc') - .limit(1) - ))[0]; - if (cache) { - const age = Date.now() - cache.createdAt.valueOf(); - const stale = cache.createdAt.valueOf() < (Date.now() - this.cacheValidMs); - this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`, { - query, digest: queryDigest, age, stale - }); + if (preferScraping) { + yield this.standaloneFallback; + yield this.googleSerp; + } + yield this.googleSerp; + if (includeSerper) { + yield this.serperGoogle; + yield this.serperGoogle; + } + } - if (!stale) { - return cache.response as any; - } - } + async cachedSearch( + variant: "web" | "news" | "images", + query: Record, + opts: CrawlerOptions, + ) { + const queryDigest = objHashMd5B64Of({ ...query, variant }); + const provider = query.provider; + Reflect.deleteProperty(query, "provider"); + const noCache = opts.noCache; + let cache; + if (!noCache) { + cache = ( + await SERPResult.fromFirestoreQuery( + SERPResult.COLLECTION.where("queryDigest", "==", queryDigest) + .orderBy("createdAt", "desc") + .limit(1), + ) + )[0]; + if (cache) { + const age = Date.now() - cache.createdAt.valueOf(); + const stale = + cache.createdAt.valueOf() < Date.now() - this.cacheValidMs; + this.logger.info( + `${stale ? "Stale cache exists" : "Cache hit"} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`, + { + query, + digest: queryDigest, + age, + stale, + }, + ); + + if (!stale) { + return cache.response as any; } - const scrappingOptions = await this.configure(opts); + } + } + const scrappingOptions = await this.configure(opts); + try { + let r: any[] | undefined; + let lastError; + outerLoop: for (const client of this.iterProviders(provider, variant)) { + const t0 = Date.now(); try { - let r: any[] | undefined; - let lastError; - outerLoop: - for (const client of this.iterProviders(provider, variant)) { - const t0 = Date.now(); - try { - switch (variant) { - case 'images': { - r = await Reflect.apply(client.imageSearch, client, [query, scrappingOptions]); - break; - } - case 'news': { - r = await Reflect.apply(client.newsSearch, client, [query, scrappingOptions]); - break; - } - case 'web': - default: { - r = await Reflect.apply(client.webSearch, client, [query, scrappingOptions]); - break; - } - } - const dt = Date.now() - t0; - this.logger.info(`Search took ${dt}ms, ${client.constructor.name}(${variant})`, { searchDt: dt, variant, client: client.constructor.name }); - break outerLoop; - } catch (err) { - lastError = err; - const dt = Date.now() - t0; - this.logger.warn(`Failed to do ${variant} search using ${client.constructor.name}`, { err, variant, searchDt: dt, }); - } + switch (variant) { + case "images": { + r = await Reflect.apply(client.imageSearch, client, [ + query, + scrappingOptions, + ]); + break; } - - if (r?.length) { - const nowDate = new Date(); - const record = SERPResult.from({ - query, - queryDigest, - response: r, - createdAt: nowDate, - expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs) - }); - this.batchedCaches.push(record); - } else if (lastError) { - throw lastError; + case "news": { + r = await Reflect.apply(client.newsSearch, client, [ + query, + scrappingOptions, + ]); + break; } - - return r; - } catch (err: any) { - if (cache) { - this.logger.warn(`Failed to fetch search result, but a stale cache is available. falling back to stale cache`, { err: marshalErrorLike(err) }); - - return cache.response as any; + case "web": + default: { + r = await Reflect.apply(client.webSearch, client, [ + query, + scrappingOptions, + ]); + break; } - - throw err; + } + const dt = Date.now() - t0; + this.logger.info( + `Search took ${dt}ms, ${client.constructor.name}(${variant})`, + { searchDt: dt, variant, client: client.constructor.name }, + ); + break outerLoop; + } catch (err) { + lastError = err; + const dt = Date.now() - t0; + this.logger.warn( + `Failed to do ${variant} search using ${client.constructor.name}`, + { err, variant, searchDt: dt }, + ); } - } + } - async assignGeneralMixin(result: Partial) { - const collectFavicon = this.threadLocal.get('collect-favicon'); + if (r?.length) { + const nowDate = new Date(); + const record = SERPResult.from({ + query, + queryDigest, + response: r, + createdAt: nowDate, + expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs), + }); + this.batchedCaches.push(record); + } else if (lastError) { + throw lastError; + } - if (collectFavicon && result.link) { - const url = new URL(result.link); - Reflect.set(result, 'favicon', await this.getFavicon(url.origin)); - } + return r; + } catch (err: any) { + if (cache) { + this.logger.warn( + `Failed to fetch search result, but a stale cache is available. falling back to stale cache`, + { err: marshalErrorLike(err) }, + ); + + return cache.response as any; + } + + throw err; } + } + + async assignGeneralMixin(result: Partial) { + const collectFavicon = this.threadLocal.get("collect-favicon"); + + if (collectFavicon && result.link) { + const url = new URL(result.link); + Reflect.set(result, "favicon", await this.getFavicon(url.origin)); + } + } } diff --git a/src/db/adaptive-crawl-task.ts b/src/db/adaptive-crawl-task.ts index 651b4d1..50998b5 100644 --- a/src/db/adaptive-crawl-task.ts +++ b/src/db/adaptive-crawl-task.ts @@ -1,86 +1,85 @@ -import { Also, Prop, parseJSONText } from 'civkit'; -import { FirestoreRecord } from '../shared/lib/firestore'; -import _ from 'lodash'; +import { Also, Prop, parseJSONText } from "civkit"; +import { FirestoreRecord } from "../shared/lib/firestore"; +import _ from "lodash"; export enum AdaptiveCrawlTaskStatus { - PENDING = 'pending', - PROCESSING = 'processing', - COMPLETED = 'completed', - FAILED = 'failed', + PENDING = "pending", + PROCESSING = "processing", + COMPLETED = "completed", + FAILED = "failed", } @Also({ - dictOf: Object + dictOf: Object, }) export class AdaptiveCrawlTask extends FirestoreRecord { - static override collectionName = 'adaptiveCrawlTasks'; + static override collectionName = "adaptiveCrawlTasks"; - override _id!: string; + override _id!: string; - @Prop({ - required: true - }) - status!: AdaptiveCrawlTaskStatus; + @Prop({ + required: true, + }) + status!: AdaptiveCrawlTaskStatus; - @Prop({ - required: true - }) - statusText!: string; + @Prop({ + required: true, + }) + statusText!: string; - @Prop() - meta!: { - useSitemap: boolean; - maxPages: number; - targetUrl: string; - }; + @Prop() + meta!: { + useSitemap: boolean; + maxPages: number; + targetUrl: string; + }; - @Prop() - urls!: string[]; + @Prop() + urls!: string[]; - @Prop() - processed!: { - [url: string]: string; - }; + @Prop() + processed!: { + [url: string]: string; + }; - @Prop() - failed!: { - [url: string]: any; - }; + @Prop() + failed!: { + [url: string]: any; + }; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; - @Prop() - finishedAt?: Date; + @Prop() + finishedAt?: Date; - @Prop() - duration?: number; + @Prop() + duration?: number; - static patchedFields = [ - 'meta', - ]; + static patchedFields = ["meta"]; - static override from(input: any) { - for (const field of this.patchedFields) { - if (typeof input[field] === 'string') { - input[field] = parseJSONText(input[field]); - } - } - - return super.from(input) as AdaptiveCrawlTask; + static override from(input: any) { + for (const field of this.patchedFields) { + if (typeof input[field] === "string") { + input[field] = parseJSONText(input[field]); + } } - override degradeForFireStore() { - const copy: any = { ...this }; + return super.from(input) as AdaptiveCrawlTask; + } - for (const field of (this.constructor as typeof AdaptiveCrawlTask).patchedFields) { - if (typeof copy[field] === 'object') { - copy[field] = JSON.stringify(copy[field]) as any; - } - } + override degradeForFireStore() { + const copy: any = { ...this }; - return copy; + for (const field of (this.constructor as typeof AdaptiveCrawlTask) + .patchedFields) { + if (typeof copy[field] === "object") { + copy[field] = JSON.stringify(copy[field]) as any; + } } - [k: string]: any; + return copy; + } + + [k: string]: any; } diff --git a/src/db/crawled.ts b/src/db/crawled.ts index 100da62..3a25faa 100644 --- a/src/db/crawled.ts +++ b/src/db/crawled.ts @@ -1,72 +1,70 @@ -import { Also, parseJSONText, Prop } from 'civkit'; -import { FirestoreRecord } from '../shared/lib/firestore'; -import _ from 'lodash'; -import type { PageSnapshot } from '../services/puppeteer'; +import { Also, parseJSONText, Prop } from "civkit"; +import { FirestoreRecord } from "../shared/lib/firestore"; +import _ from "lodash"; +import type { PageSnapshot } from "../services/puppeteer"; @Also({ - dictOf: Object + dictOf: Object, }) export class Crawled extends FirestoreRecord { - static override collectionName = 'crawled'; + static override collectionName = "crawled"; - override _id!: string; + override _id!: string; - @Prop({ - required: true - }) - url!: string; + @Prop({ + required: true, + }) + url!: string; - @Prop({ - required: true - }) - urlPathDigest!: string; + @Prop({ + required: true, + }) + urlPathDigest!: string; - @Prop() - htmlSignificantlyModifiedByJs?: boolean; + @Prop() + htmlSignificantlyModifiedByJs?: boolean; - @Prop() - snapshot?: PageSnapshot & { screenshot: never; pageshot: never; }; + @Prop() + snapshot?: PageSnapshot & { screenshot: never; pageshot: never }; - @Prop() - screenshotAvailable?: boolean; + @Prop() + screenshotAvailable?: boolean; - @Prop() - pageshotAvailable?: boolean; + @Prop() + pageshotAvailable?: boolean; - @Prop() - snapshotAvailable?: boolean; + @Prop() + snapshotAvailable?: boolean; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; - @Prop() - expireAt!: Date; + @Prop() + expireAt!: Date; - static patchedFields = [ - 'snapshot' - ]; + static patchedFields = ["snapshot"]; - static override from(input: any) { - for (const field of this.patchedFields) { - if (typeof input[field] === 'string') { - input[field] = parseJSONText(input[field]); - } - } - - return super.from(input) as Crawled; + static override from(input: any) { + for (const field of this.patchedFields) { + if (typeof input[field] === "string") { + input[field] = parseJSONText(input[field]); + } } - override degradeForFireStore() { - const copy: any = { ...this }; + return super.from(input) as Crawled; + } - for (const field of (this.constructor as typeof Crawled).patchedFields) { - if (typeof copy[field] === 'object') { - copy[field] = JSON.stringify(copy[field]) as any; - } - } + override degradeForFireStore() { + const copy: any = { ...this }; - return copy; + for (const field of (this.constructor as typeof Crawled).patchedFields) { + if (typeof copy[field] === "object") { + copy[field] = JSON.stringify(copy[field]) as any; + } } - [k: string]: any; + return copy; + } + + [k: string]: any; } diff --git a/src/db/domain-blockade.ts b/src/db/domain-blockade.ts index 72955a2..29d710a 100644 --- a/src/db/domain-blockade.ts +++ b/src/db/domain-blockade.ts @@ -1,30 +1,30 @@ -import { Also, Prop } from 'civkit'; -import { FirestoreRecord } from '../shared/lib/firestore'; +import { Also, Prop } from "civkit"; +import { FirestoreRecord } from "../shared/lib/firestore"; @Also({ - dictOf: Object + dictOf: Object, }) export class DomainBlockade extends FirestoreRecord { - static override collectionName = 'domainBlockades'; + static override collectionName = "domainBlockades"; - override _id!: string; + override _id!: string; - @Prop({ - required: true - }) - domain!: string; + @Prop({ + required: true, + }) + domain!: string; - @Prop({ required: true }) - triggerReason!: string; + @Prop({ required: true }) + triggerReason!: string; - @Prop() - triggerUrl?: string; + @Prop() + triggerUrl?: string; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; - @Prop() - expireAt?: Date; + @Prop() + expireAt?: Date; - [k: string]: any; + [k: string]: any; } diff --git a/src/db/domain-profile.ts b/src/db/domain-profile.ts index 3ee9476..cca19ec 100644 --- a/src/db/domain-profile.ts +++ b/src/db/domain-profile.ts @@ -1,31 +1,31 @@ -import { Also, Prop } from 'civkit'; -import { FirestoreRecord } from '../shared/lib/firestore'; -import { ENGINE_TYPE } from '../dto/crawler-options'; +import { Also, Prop } from "civkit"; +import { FirestoreRecord } from "../shared/lib/firestore"; +import { ENGINE_TYPE } from "../dto/crawler-options"; @Also({ - dictOf: Object + dictOf: Object, }) export class DomainProfile extends FirestoreRecord { - static override collectionName = 'domainProfiles'; + static override collectionName = "domainProfiles"; - override _id!: string; + override _id!: string; - @Prop({ - required: true - }) - path!: string; + @Prop({ + required: true, + }) + path!: string; - @Prop() - triggerUrl?: string; + @Prop() + triggerUrl?: string; - @Prop({ required: true, type: ENGINE_TYPE }) - engine!: string; + @Prop({ required: true, type: ENGINE_TYPE }) + engine!: string; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; - @Prop() - expireAt?: Date; + @Prop() + expireAt?: Date; - [k: string]: any; + [k: string]: any; } diff --git a/src/db/img-alt.ts b/src/db/img-alt.ts index a1221ba..ebf561a 100644 --- a/src/db/img-alt.ts +++ b/src/db/img-alt.ts @@ -1,42 +1,42 @@ -import { Also, Prop } from 'civkit'; -import { FirestoreRecord } from '../shared/lib/firestore'; -import _ from 'lodash'; +import { Also, Prop } from "civkit"; +import { FirestoreRecord } from "../shared/lib/firestore"; +import _ from "lodash"; @Also({ - dictOf: Object + dictOf: Object, }) export class ImgAlt extends FirestoreRecord { - static override collectionName = 'imgAlts'; + static override collectionName = "imgAlts"; - override _id!: string; + override _id!: string; - @Prop({ - required: true - }) - src!: string; + @Prop({ + required: true, + }) + src!: string; - @Prop({ - required: true - }) - urlDigest!: string; + @Prop({ + required: true, + }) + urlDigest!: string; - @Prop() - width?: number; + @Prop() + width?: number; - @Prop() - height?: number; + @Prop() + height?: number; - @Prop() - generatedAlt?: string; + @Prop() + generatedAlt?: string; - @Prop() - originalAlt?: string; + @Prop() + originalAlt?: string; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; - @Prop() - expireAt?: Date; + @Prop() + expireAt?: Date; - [k: string]: any; + [k: string]: any; } diff --git a/src/db/pdf.ts b/src/db/pdf.ts index bb088e9..6cf5b10 100644 --- a/src/db/pdf.ts +++ b/src/db/pdf.ts @@ -1,65 +1,63 @@ -import { Also, Prop, parseJSONText } from 'civkit'; -import { FirestoreRecord } from '../shared/lib/firestore'; -import _ from 'lodash'; +import { Also, Prop, parseJSONText } from "civkit"; +import { FirestoreRecord } from "../shared/lib/firestore"; +import _ from "lodash"; @Also({ - dictOf: Object + dictOf: Object, }) export class PDFContent extends FirestoreRecord { - static override collectionName = 'pdfs'; + static override collectionName = "pdfs"; - override _id!: string; + override _id!: string; - @Prop({ - required: true - }) - src!: string; + @Prop({ + required: true, + }) + src!: string; - @Prop({ - required: true - }) - urlDigest!: string; + @Prop({ + required: true, + }) + urlDigest!: string; - @Prop() - meta?: { [k: string]: any; }; + @Prop() + meta?: { [k: string]: any }; - @Prop() - text?: string; + @Prop() + text?: string; - @Prop() - content?: string; + @Prop() + content?: string; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; - @Prop() - expireAt?: Date; + @Prop() + expireAt?: Date; - static patchedFields = [ - 'meta' - ]; + static patchedFields = ["meta"]; - static override from(input: any) { - for (const field of this.patchedFields) { - if (typeof input[field] === 'string') { - input[field] = parseJSONText(input[field]); - } - } - - return super.from(input) as PDFContent; + static override from(input: any) { + for (const field of this.patchedFields) { + if (typeof input[field] === "string") { + input[field] = parseJSONText(input[field]); + } } - override degradeForFireStore() { - const copy: any = { ...this }; + return super.from(input) as PDFContent; + } - for (const field of (this.constructor as typeof PDFContent).patchedFields) { - if (typeof copy[field] === 'object') { - copy[field] = JSON.stringify(copy[field]) as any; - } - } + override degradeForFireStore() { + const copy: any = { ...this }; - return copy; + for (const field of (this.constructor as typeof PDFContent).patchedFields) { + if (typeof copy[field] === "object") { + copy[field] = JSON.stringify(copy[field]) as any; + } } - [k: string]: any; + return copy; + } + + [k: string]: any; } diff --git a/src/db/searched.ts b/src/db/searched.ts index aa05ad0..855407c 100644 --- a/src/db/searched.ts +++ b/src/db/searched.ts @@ -1,68 +1,66 @@ -import { Also, parseJSONText, Prop } from 'civkit'; -import { FirestoreRecord } from '../shared/lib/firestore'; -import _ from 'lodash'; +import { Also, parseJSONText, Prop } from "civkit"; +import { FirestoreRecord } from "../shared/lib/firestore"; +import _ from "lodash"; @Also({ - dictOf: Object + dictOf: Object, }) export class SearchResult extends FirestoreRecord { - static override collectionName = 'searchResults'; + static override collectionName = "searchResults"; - override _id!: string; + override _id!: string; - @Prop({ - required: true - }) - query!: any; + @Prop({ + required: true, + }) + query!: any; - @Prop({ - required: true - }) - queryDigest!: string; + @Prop({ + required: true, + }) + queryDigest!: string; - @Prop() - response?: any; + @Prop() + response?: any; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; - @Prop() - expireAt?: Date; + @Prop() + expireAt?: Date; - [k: string]: any; + [k: string]: any; - static patchedFields = [ - 'query', - 'response', - ]; + static patchedFields = ["query", "response"]; - static override from(input: any) { - for (const field of this.patchedFields) { - if (typeof input[field] === 'string') { - input[field] = parseJSONText(input[field]); - } - } - - return super.from(input) as SearchResult; + static override from(input: any) { + for (const field of this.patchedFields) { + if (typeof input[field] === "string") { + input[field] = parseJSONText(input[field]); + } } - override degradeForFireStore() { - const copy: any = { ...this }; + return super.from(input) as SearchResult; + } - for (const field of (this.constructor as typeof SearchResult).patchedFields) { - if (typeof copy[field] === 'object') { - copy[field] = JSON.stringify(copy[field]) as any; - } - } + override degradeForFireStore() { + const copy: any = { ...this }; - return copy; + for (const field of (this.constructor as typeof SearchResult) + .patchedFields) { + if (typeof copy[field] === "object") { + copy[field] = JSON.stringify(copy[field]) as any; + } } + + return copy; + } } export class SerperSearchResult extends SearchResult { - static override collectionName = 'serperSearchResults'; + static override collectionName = "serperSearchResults"; } export class SERPResult extends SearchResult { - static override collectionName = 'SERPResults'; -} \ No newline at end of file + static override collectionName = "SERPResults"; +} diff --git a/src/dto/adaptive-crawler-options.ts b/src/dto/adaptive-crawler-options.ts index 966fef6..81e21ec 100644 --- a/src/dto/adaptive-crawler-options.ts +++ b/src/dto/adaptive-crawler-options.ts @@ -1,63 +1,64 @@ -import { Also, AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from 'civkit'; +import { Also, AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from "civkit"; type HeaderReadableRequest = { - get(name: string): string | undefined; + get(name: string): string | undefined; }; - @Also({ - openapi: { - operation: { - parameters: { - 'X-Use-Sitemap': { - description: 'Use sitemap to crawl the website.', - in: 'header', - schema: { type: 'string' } - }, - 'X-Max-Depth': { - description: 'Max deep level to crawl.', - in: 'header', - schema: { type: 'string' } - }, - 'X-Max-Pages': { - description: 'Max number of pages to crawl.', - in: 'header', - schema: { type: 'string' } - }, - } - } - } + openapi: { + operation: { + parameters: { + "X-Use-Sitemap": { + description: "Use sitemap to crawl the website.", + in: "header", + schema: { type: "string" }, + }, + "X-Max-Depth": { + description: "Max deep level to crawl.", + in: "header", + schema: { type: "string" }, + }, + "X-Max-Pages": { + description: "Max number of pages to crawl.", + in: "header", + schema: { type: "string" }, + }, + }, + }, + }, }) export class AdaptiveCrawlerOptions extends AutoCastable { - @Prop({ - default: true, - desc: 'Use sitemap to crawl the website.', - }) - useSitemap!: boolean; + @Prop({ + default: true, + desc: "Use sitemap to crawl the website.", + }) + useSitemap!: boolean; - @Prop({ - default: 10, - desc: 'Max number of pages to crawl.', - validate: (v: number) => v >= 1 && v <= 100, - }) - maxPages!: number; + @Prop({ + default: 10, + desc: "Max number of pages to crawl.", + validate: (v: number) => v >= 1 && v <= 100, + }) + maxPages!: number; - static override from(input: any) { - const instance = super.from(input) as AdaptiveCrawlerOptions; - const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as { - req: HeaderReadableRequest, - } | undefined; - - let maxPages = parseInt(ctx?.req.get('x-max-pages') || ''); - if (!isNaN(maxPages) && maxPages > 0) { - instance.maxPages = maxPages <= 100 ? maxPages : 100; + static override from(input: any) { + const instance = super.from(input) as AdaptiveCrawlerOptions; + const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as + | { + req: HeaderReadableRequest; } + | undefined; - const useSitemap = ctx?.req.get('x-use-sitemap'); - if (useSitemap !== undefined) { - instance.useSitemap = Boolean(useSitemap); - } - - return instance; + let maxPages = parseInt(ctx?.req.get("x-max-pages") || ""); + if (!isNaN(maxPages) && maxPages > 0) { + instance.maxPages = maxPages <= 100 ? maxPages : 100; } + + const useSitemap = ctx?.req.get("x-use-sitemap"); + if (useSitemap !== undefined) { + instance.useSitemap = Boolean(useSitemap); + } + + return instance; + } } diff --git a/src/dto/auth.ts b/src/dto/auth.ts index a41d099..87bce83 100644 --- a/src/dto/auth.ts +++ b/src/dto/auth.ts @@ -1,164 +1,170 @@ import { - Also, AuthenticationRequiredError, - AutoCastable, RPC_CALL_ENVIRONMENT, -} from 'civkit/civ-rpc'; -import { htmlEscape } from 'civkit/escape'; -import { Prop } from 'civkit'; + Also, + AuthenticationRequiredError, + AutoCastable, + RPC_CALL_ENVIRONMENT, +} from "civkit/civ-rpc"; +import { htmlEscape } from "civkit/escape"; +import { Prop } from "civkit"; -import type { Context } from 'koa'; +import type { Context } from "koa"; -import { InjectProperty } from '../services/registry'; -import { AsyncLocalContext } from '../services/async-context'; -import { RateLimitDesc } from '../shared/services/rate-limit'; +import { InjectProperty } from "../services/registry"; +import { AsyncLocalContext } from "../services/async-context"; +import { RateLimitDesc } from "../shared/services/rate-limit"; -const ANONYMOUS_USER_ID = 'anonymous'; +const ANONYMOUS_USER_ID = "anonymous"; export class ApiTokenAccount extends AutoCastable { - @Prop() - user_id = ANONYMOUS_USER_ID; + @Prop() + user_id = ANONYMOUS_USER_ID; - @Prop() - full_name = 'Anonymous'; + @Prop() + full_name = "Anonymous"; - @Prop() - wallet = { - total_balance: Number.MAX_SAFE_INTEGER, - }; + @Prop() + wallet = { + total_balance: Number.MAX_SAFE_INTEGER, + }; - @Prop() - metadata: Record = { - speed_level: '0', - }; + @Prop() + metadata: Record = { + speed_level: "0", + }; - @Prop() - customRateLimits?: Record; + @Prop() + customRateLimits?: Record; - @Prop() - lastSyncedAt = new Date(); + @Prop() + lastSyncedAt = new Date(); } @Also({ - openapi: { - operation: { - parameters: { - 'Authorization': { - description: htmlEscape`Optional API token for compatibility.\n\n` + - htmlEscape`- Member of \n\n` + - `- Authorization: Bearer {YOUR_XREAD_TOKEN}`, - in: 'header', - schema: { - anyOf: [ - { type: 'string', format: 'token' }, - ], - }, - }, - }, + openapi: { + operation: { + parameters: { + Authorization: { + description: + htmlEscape`Optional API token for compatibility.\n\n` + + htmlEscape`- Member of \n\n` + + `- Authorization: Bearer {YOUR_XREAD_TOKEN}`, + in: "header", + schema: { + anyOf: [{ type: "string", format: "token" }], + }, }, + }, }, + }, }) export class AuthDTO extends AutoCastable { - uid?: string; - bearerToken?: string; - user?: ApiTokenAccount; + uid?: string; + bearerToken?: string; + user?: ApiTokenAccount; - @InjectProperty(AsyncLocalContext) - ctxMgr!: AsyncLocalContext; + @InjectProperty(AsyncLocalContext) + ctxMgr!: AsyncLocalContext; - static override from(input: any) { - const instance = super.from(input) as AuthDTO; - const ctx = input[RPC_CALL_ENVIRONMENT] as Context | undefined; + static override from(input: any) { + const instance = super.from(input) as AuthDTO; + const ctx = input[RPC_CALL_ENVIRONMENT] as Context | undefined; - if (ctx) { - const authorization = ctx.get('authorization'); - if (authorization) { - instance.bearerToken = authorization.split(' ')[1] || authorization; - } - } - - if (!instance.bearerToken && input._token) { - instance.bearerToken = input._token; - } - - return instance; + if (ctx) { + const authorization = ctx.get("authorization"); + if (authorization) { + instance.bearerToken = authorization.split(" ")[1] || authorization; + } } - protected buildLocalAccount() { - const tokenSuffix = this.bearerToken ? this.bearerToken.slice(-8) : ''; - const userId = this.bearerToken ? `token:${tokenSuffix || 'local'}` : ANONYMOUS_USER_ID; - - return ApiTokenAccount.from({ - user_id: userId, - full_name: this.bearerToken ? 'Local Token User' : 'Anonymous', - wallet: { - total_balance: Number.MAX_SAFE_INTEGER, - }, - metadata: { - speed_level: this.bearerToken ? '1' : '0', - }, - lastSyncedAt: new Date(), - }); + if (!instance.bearerToken && input._token) { + instance.bearerToken = input._token; } - async getBrief() { - const account = this.buildLocalAccount(); - this.user = account; - this.uid = this.bearerToken ? account.user_id : undefined; + return instance; + } - return account; + protected buildLocalAccount() { + const tokenSuffix = this.bearerToken ? this.bearerToken.slice(-8) : ""; + const userId = this.bearerToken + ? `token:${tokenSuffix || "local"}` + : ANONYMOUS_USER_ID; + + return ApiTokenAccount.from({ + user_id: userId, + full_name: this.bearerToken ? "Local Token User" : "Anonymous", + wallet: { + total_balance: Number.MAX_SAFE_INTEGER, + }, + metadata: { + speed_level: this.bearerToken ? "1" : "0", + }, + lastSyncedAt: new Date(), + }); + } + + async getBrief() { + const account = this.buildLocalAccount(); + this.user = account; + this.uid = this.bearerToken ? account.user_id : undefined; + + return account; + } + + async reportUsage( + _tokenCount: number, + _mdl: string, + _endpoint: string = "/encode", + ) { + return undefined; + } + + async solveUID() { + if (this.uid) { + this.ctxMgr.set("uid", this.uid); + return this.uid; } - async reportUsage(_tokenCount: number, _mdl: string, _endpoint: string = '/encode') { - return undefined; + if (this.bearerToken) { + await this.getBrief(); + this.ctxMgr.set("uid", this.uid); + return this.uid; } - async solveUID() { - if (this.uid) { - this.ctxMgr.set('uid', this.uid); - return this.uid; - } + return undefined; + } - if (this.bearerToken) { - await this.getBrief(); - this.ctxMgr.set('uid', this.uid); - return this.uid; - } + async assertUID() { + const uid = await this.solveUID(); - return undefined; + if (!uid) { + throw new AuthenticationRequiredError("Authentication failed"); } - async assertUID() { - const uid = await this.solveUID(); + return uid; + } - if (!uid) { - throw new AuthenticationRequiredError('Authentication failed'); - } - - return uid; + async assertUser() { + if (this.user) { + return this.user; } - async assertUser() { - if (this.user) { - return this.user; - } + return this.getBrief(); + } - return this.getBrief(); + async assertTier(_n: number, _feature?: string) { + return true; + } + + getRateLimits(...tags: string[]) { + const descs = tags + .map((tag) => this.user?.customRateLimits?.[tag] || []) + .flat() + .filter((item) => item.isEffective()); + + if (descs.length) { + return descs; } - async assertTier(_n: number, _feature?: string) { - return true; - } - - getRateLimits(...tags: string[]) { - const descs = tags - .map((tag) => this.user?.customRateLimits?.[tag] || []) - .flat() - .filter((item) => item.isEffective()); - - if (descs.length) { - return descs; - } - - return undefined; - } + return undefined; + } } - diff --git a/src/dto/crawler-options.ts b/src/dto/crawler-options.ts index e411470..39871a9 100644 --- a/src/dto/crawler-options.ts +++ b/src/dto/crawler-options.ts @@ -1,727 +1,813 @@ -import { Also, AutoCastable, ParamValidationError, Prop, RPC_CALL_ENVIRONMENT } from 'civkit/civ-rpc'; -import { FancyFile } from 'civkit/fancy-file'; -import { Cookie, parseString as parseSetCookieString } from 'set-cookie-parser'; -import { Context } from '../services/registry'; -import { TurnDownTweakableOptions } from './turndown-tweakable-options'; -import type { PageSnapshot } from '../services/puppeteer'; +import { + Also, + AutoCastable, + ParamValidationError, + Prop, + RPC_CALL_ENVIRONMENT, +} from "civkit/civ-rpc"; +import { FancyFile } from "civkit/fancy-file"; +import { Cookie, parseString as parseSetCookieString } from "set-cookie-parser"; +import { Context } from "../services/registry"; +import { TurnDownTweakableOptions } from "./turndown-tweakable-options"; +import type { PageSnapshot } from "../services/puppeteer"; export enum CONTENT_FORMAT { - CONTENT = 'content', - MARKDOWN = 'markdown', - HTML = 'html', - TEXT = 'text', - PAGESHOT = 'pageshot', - SCREENSHOT = 'screenshot', - VLM = 'vlm', - XREAD_LM = 'xreadlm-v2', + CONTENT = "content", + MARKDOWN = "markdown", + HTML = "html", + TEXT = "text", + PAGESHOT = "pageshot", + SCREENSHOT = "screenshot", + VLM = "vlm", + XREAD_LM = "xreadlm-v2", } export enum ENGINE_TYPE { - AUTO = 'auto', - BROWSER = 'browser', - CURL = 'curl', - CF_BROWSER_RENDERING = 'cf-browser-rendering', + AUTO = "auto", + BROWSER = "browser", + CURL = "curl", + CF_BROWSER_RENDERING = "cf-browser-rendering", } export enum RESPOND_TIMING { - HTML = 'html', - VISIBLE_CONTENT = 'visible-content', - MUTATION_IDLE = 'mutation-idle', - RESOURCE_IDLE = 'resource-idle', - MEDIA_IDLE = 'media-idle', - NETWORK_IDLE = 'network-idle', + HTML = "html", + VISIBLE_CONTENT = "visible-content", + MUTATION_IDLE = "mutation-idle", + RESOURCE_IDLE = "resource-idle", + MEDIA_IDLE = "media-idle", + NETWORK_IDLE = "network-idle", } const CONTENT_FORMAT_VALUES = new Set(Object.values(CONTENT_FORMAT)); -export const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const; +export const IMAGE_RETENTION_MODES = [ + "none", + "all", + "alt", + "all_p", + "alt_p", +] as const; const IMAGE_RETENTION_MODE_VALUES = new Set(IMAGE_RETENTION_MODES); -export const BASE_URL_MODES = ['initial', 'final'] as const; +export const BASE_URL_MODES = ["initial", "final"] as const; const BASE_URL_MODE_VALUES = new Set(BASE_URL_MODES); class Viewport extends AutoCastable { - @Prop({ - default: 1024 - }) - width!: number; - @Prop({ - default: 1024 - }) - height!: number; - @Prop() - deviceScaleFactor?: number; - @Prop() - isMobile?: boolean; - @Prop() - isLandscape?: boolean; - @Prop() - hasTouch?: boolean; + @Prop({ + default: 1024, + }) + width!: number; + @Prop({ + default: 1024, + }) + height!: number; + @Prop() + deviceScaleFactor?: number; + @Prop() + isMobile?: boolean; + @Prop() + isLandscape?: boolean; + @Prop() + hasTouch?: boolean; } @Also({ - openapi: { - operation: { - parameters: { - 'Accept': { - description: `Specifies your preference for the response format.\n\n` + - `Supported formats: \n` + - `- text/event-stream\n` + - `- application/json or text/json\n` + - `- text/plain` - , - in: 'header', - schema: { type: 'string' } - }, - 'X-Cache-Tolerance': { - description: `Sets internal cache tolerance in seconds if this header is specified with a integer.`, - in: 'header', - schema: { type: 'string' } - }, - 'X-No-Cache': { - description: `Ignores internal cache if this header is specified with a value.\n\nEquivalent to X-Cache-Tolerance: 0`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Respond-With': { - description: `Specifies the (non-default) form of the crawled data you prefer.\n\n` + - `Supported formats: \n` + - `- markdown\n` + - `- html\n` + - `- text\n` + - `- pageshot\n` + - `- screenshot\n` + - `- content\n` + - `- any combination of the above\n` + - `- xreadlm-v2\n` + - `- vlm\n\n` + - `Default: content\n` - , - in: 'header', - schema: { type: 'string' } - }, - 'X-Wait-For-Selector': { - description: `Specifies a CSS selector to wait for the appearance of such an element before returning.\n\n` + - 'Example: `X-Wait-For-Selector: .content-block`\n' - , - in: 'header', - schema: { type: 'string' } - }, - 'X-Target-Selector': { - description: `Specifies a CSS selector for return target instead of the full html.\n\n` + - 'Implies `X-Wait-For-Selector: (same selector)`' - , - in: 'header', - schema: { type: 'string' } - }, - 'X-Remove-Selector': { - description: `Specifies a CSS selector to remove elements from the full html.\n\n` + - 'Example `X-Remove-Selector: nav`' - , - in: 'header', - schema: { type: 'string' } - }, - 'X-Keep-Img-Data-Url': { - description: `Keep data-url as it instead of transforming them to object-url. (Only applicable when targeting markdown format)\n\n` + - 'Example `X-Keep-Img-Data-Url: true`' - , - in: 'header', - schema: { type: 'string' } - }, - 'X-Proxy-Url': { - description: `Specifies your custom proxy if you prefer to use one.\n\n` + - `Supported protocols: \n` + - `- http\n` + - `- https\n` + - `- socks4\n` + - `- socks5\n\n` + - `For authentication, https://user:pass@host:port`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Proxy': { - description: `Use a proxy server provided by us.\n\nOptionally specify two-letter country code.`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Robots-Txt': { - description: `Load and conform to the respective robot.txt on the target origin.\n\nOptionally specify a bot UA to check against.\n\n`, - in: 'header', - schema: { type: 'string' } - }, - 'DNT': { - description: `When set to 1, prevent the result of this request to be cached in the system.\n\n`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Set-Cookie': { - description: `Sets cookie(s) to the headless browser for your request. \n\n` + - `Syntax is the same with standard Set-Cookie`, - in: 'header', - schema: { type: 'string' } - }, - 'X-With-Generated-Alt': { - description: `Enable automatic alt-text generating for images without an meaningful alt-text.\n\n` + - `Note: Does not work when \`X-Respond-With\` is specified`, - in: 'header', - schema: { type: 'string' } - }, - 'X-With-Images-Summary': { - description: `Enable dedicated summary section for images on the page.`, - in: 'header', - schema: { type: 'string' } - }, - 'X-With-links-Summary': { - description: `Enable dedicated summary section for hyper links on the page.`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Retain-Images': { - description: `Image retention modes.\n\n` + - `Supported modes: \n` + - `- all: all images\n` + - `- none: no images\n` + - `- alt: only alt text\n` + - `- all_p: all images and with generated alt text\n` + - `- alt_p: only alt text and with generated alt\n\n`, - in: 'header', - schema: { type: 'string' } - }, - 'X-With-Iframe': { - description: `Enable filling iframe contents into main. (violates standards)`, - in: 'header', - schema: { type: 'string' } - }, - 'X-With-Shadow-Dom': { - description: `Enable filling shadow dom contents into main. (violates standards)`, - in: 'header', - schema: { type: 'string' } - }, - 'X-User-Agent': { - description: `Override User-Agent.`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Timeout': { - description: `Specify timeout in seconds. Max 180.`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Locale': { - description: 'Specify browser locale for the page.', - in: 'header', - schema: { type: 'string' } - }, - 'X-Referer': { - description: 'Specify referer for the page.', - in: 'header', - schema: { type: 'string' } - }, - 'X-Token-Budget': { - description: 'Specify a budget in tokens.\n\nIf the resulting token cost exceeds the budget, the request is rejected.', - in: 'header', - schema: { type: 'string' } - }, - 'X-Respond-Timing': { - description: `Explicitly specify the respond timing. One of the following:\n\n` + - `- html: directly return unrendered HTML\n` + - `- visible-content: return immediately when any content becomes available\n` + - `- mutation-idle: wait for DOM mutations to settle and remain unchanged for at least 0.2s\n` + - `- resource-idle: wait for no additional resources that would affect page logic and content has SUCCEEDED loading in 0.5s\n` + - `- media-idle: wait for no additional resources, including media resources, has SUCCEEDED loading in 0.5s\n` + - `- network-idle: wait for full load of webpage, also known as networkidle0.\n\n`, - in: 'header', - schema: { type: 'string' } - }, - 'X-Engine': { - description: 'Specify the engine to use for crawling.\n\nSupported: browser, direct, cf-browser-rendering', - in: 'header', - schema: { type: 'string' } - }, - 'X-Base': { - description: 'Select base modes of relative URLs.\n\nSupported: initial, final', - in: 'header', - schema: { type: 'string' } - }, - 'X-Md-Heading-Style': { - description: 'Heading style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: setext, atx', - in: 'header', - schema: { type: 'string' } - }, - 'X-Md-Hr': { - description: 'Hr text of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).', - in: 'header', - schema: { type: 'string' } - }, - 'X-Md-Bullet-List-Marker': { - description: 'Bullet list marker of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: -, +, *', - in: 'header', - schema: { type: 'string' } - }, - 'X-Md-Em-Delimiter': { - description: 'Em delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: _, *', - in: 'header', - schema: { type: 'string' } - }, - 'X-Md-Strong-Delimiter': { - description: 'Strong delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: **, __', - in: 'header', - schema: { type: 'string' } - }, - 'X-Md-Link-Style': { - description: 'Link style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: inlined, referenced, discarded', - in: 'header', - schema: { type: 'string' } - }, - 'X-Md-Link-Reference-Style': { - description: 'Link reference style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: full, collapsed, shortcut, discarded', - in: 'header', - schema: { type: 'string' } - }, - } - } - } + openapi: { + operation: { + parameters: { + Accept: { + description: + `Specifies your preference for the response format.\n\n` + + `Supported formats: \n` + + `- text/event-stream\n` + + `- application/json or text/json\n` + + `- text/plain`, + in: "header", + schema: { type: "string" }, + }, + "X-Cache-Tolerance": { + description: `Sets internal cache tolerance in seconds if this header is specified with a integer.`, + in: "header", + schema: { type: "string" }, + }, + "X-No-Cache": { + description: `Ignores internal cache if this header is specified with a value.\n\nEquivalent to X-Cache-Tolerance: 0`, + in: "header", + schema: { type: "string" }, + }, + "X-Respond-With": { + description: + `Specifies the (non-default) form of the crawled data you prefer.\n\n` + + `Supported formats: \n` + + `- markdown\n` + + `- html\n` + + `- text\n` + + `- pageshot\n` + + `- screenshot\n` + + `- content\n` + + `- any combination of the above\n` + + `- xreadlm-v2\n` + + `- vlm\n\n` + + `Default: content\n`, + in: "header", + schema: { type: "string" }, + }, + "X-Wait-For-Selector": { + description: + `Specifies a CSS selector to wait for the appearance of such an element before returning.\n\n` + + "Example: `X-Wait-For-Selector: .content-block`\n", + in: "header", + schema: { type: "string" }, + }, + "X-Target-Selector": { + description: + `Specifies a CSS selector for return target instead of the full html.\n\n` + + "Implies `X-Wait-For-Selector: (same selector)`", + in: "header", + schema: { type: "string" }, + }, + "X-Remove-Selector": { + description: + `Specifies a CSS selector to remove elements from the full html.\n\n` + + "Example `X-Remove-Selector: nav`", + in: "header", + schema: { type: "string" }, + }, + "X-Keep-Img-Data-Url": { + description: + `Keep data-url as it instead of transforming them to object-url. (Only applicable when targeting markdown format)\n\n` + + "Example `X-Keep-Img-Data-Url: true`", + in: "header", + schema: { type: "string" }, + }, + "X-Proxy-Url": { + description: + `Specifies your custom proxy if you prefer to use one.\n\n` + + `Supported protocols: \n` + + `- http\n` + + `- https\n` + + `- socks4\n` + + `- socks5\n\n` + + `For authentication, https://user:pass@host:port`, + in: "header", + schema: { type: "string" }, + }, + "X-Proxy": { + description: `Use a proxy server provided by us.\n\nOptionally specify two-letter country code.`, + in: "header", + schema: { type: "string" }, + }, + "X-Robots-Txt": { + description: `Load and conform to the respective robot.txt on the target origin.\n\nOptionally specify a bot UA to check against.\n\n`, + in: "header", + schema: { type: "string" }, + }, + DNT: { + description: `When set to 1, prevent the result of this request to be cached in the system.\n\n`, + in: "header", + schema: { type: "string" }, + }, + "X-Set-Cookie": { + description: + `Sets cookie(s) to the headless browser for your request. \n\n` + + `Syntax is the same with standard Set-Cookie`, + in: "header", + schema: { type: "string" }, + }, + "X-With-Generated-Alt": { + description: + `Enable automatic alt-text generating for images without an meaningful alt-text.\n\n` + + `Note: Does not work when \`X-Respond-With\` is specified`, + in: "header", + schema: { type: "string" }, + }, + "X-With-Images-Summary": { + description: `Enable dedicated summary section for images on the page.`, + in: "header", + schema: { type: "string" }, + }, + "X-With-links-Summary": { + description: `Enable dedicated summary section for hyper links on the page.`, + in: "header", + schema: { type: "string" }, + }, + "X-Retain-Images": { + description: + `Image retention modes.\n\n` + + `Supported modes: \n` + + `- all: all images\n` + + `- none: no images\n` + + `- alt: only alt text\n` + + `- all_p: all images and with generated alt text\n` + + `- alt_p: only alt text and with generated alt\n\n`, + in: "header", + schema: { type: "string" }, + }, + "X-With-Iframe": { + description: `Enable filling iframe contents into main. (violates standards)`, + in: "header", + schema: { type: "string" }, + }, + "X-With-Shadow-Dom": { + description: `Enable filling shadow dom contents into main. (violates standards)`, + in: "header", + schema: { type: "string" }, + }, + "X-User-Agent": { + description: `Override User-Agent.`, + in: "header", + schema: { type: "string" }, + }, + "X-Timeout": { + description: `Specify timeout in seconds. Max 180.`, + in: "header", + schema: { type: "string" }, + }, + "X-Locale": { + description: "Specify browser locale for the page.", + in: "header", + schema: { type: "string" }, + }, + "X-Referer": { + description: "Specify referer for the page.", + in: "header", + schema: { type: "string" }, + }, + "X-Token-Budget": { + description: + "Specify a budget in tokens.\n\nIf the resulting token cost exceeds the budget, the request is rejected.", + in: "header", + schema: { type: "string" }, + }, + "X-Respond-Timing": { + description: + `Explicitly specify the respond timing. One of the following:\n\n` + + `- html: directly return unrendered HTML\n` + + `- visible-content: return immediately when any content becomes available\n` + + `- mutation-idle: wait for DOM mutations to settle and remain unchanged for at least 0.2s\n` + + `- resource-idle: wait for no additional resources that would affect page logic and content has SUCCEEDED loading in 0.5s\n` + + `- media-idle: wait for no additional resources, including media resources, has SUCCEEDED loading in 0.5s\n` + + `- network-idle: wait for full load of webpage, also known as networkidle0.\n\n`, + in: "header", + schema: { type: "string" }, + }, + "X-Engine": { + description: + "Specify the engine to use for crawling.\n\nSupported: browser, direct, cf-browser-rendering", + in: "header", + schema: { type: "string" }, + }, + "X-Base": { + description: + "Select base modes of relative URLs.\n\nSupported: initial, final", + in: "header", + schema: { type: "string" }, + }, + "X-Md-Heading-Style": { + description: + "Heading style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: setext, atx", + in: "header", + schema: { type: "string" }, + }, + "X-Md-Hr": { + description: + "Hr text of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).", + in: "header", + schema: { type: "string" }, + }, + "X-Md-Bullet-List-Marker": { + description: + "Bullet list marker of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: -, +, *", + in: "header", + schema: { type: "string" }, + }, + "X-Md-Em-Delimiter": { + description: + "Em delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: _, *", + in: "header", + schema: { type: "string" }, + }, + "X-Md-Strong-Delimiter": { + description: + "Strong delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: **, __", + in: "header", + schema: { type: "string" }, + }, + "X-Md-Link-Style": { + description: + "Link style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: inlined, referenced, discarded", + in: "header", + schema: { type: "string" }, + }, + "X-Md-Link-Reference-Style": { + description: + "Link reference style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: full, collapsed, shortcut, discarded", + in: "header", + schema: { type: "string" }, + }, + }, + }, + }, }) export class CrawlerOptions extends AutoCastable { + @Prop() + url?: string; - @Prop() - url?: string; + @Prop() + html?: string; - @Prop() - html?: string; + @Prop({ + type: BASE_URL_MODE_VALUES, + default: "initial", + }) + base?: (typeof BASE_URL_MODES)[number]; - @Prop({ - type: BASE_URL_MODE_VALUES, - default: 'initial', - }) - base?: typeof BASE_URL_MODES[number]; + @Prop({ + desc: "Base64 encoded PDF.", + type: [FancyFile, String], + }) + pdf?: FancyFile | string; - @Prop({ - desc: 'Base64 encoded PDF.', - type: [FancyFile, String] - }) - pdf?: FancyFile | string; + @Prop({ + default: CONTENT_FORMAT.CONTENT, + type: [CONTENT_FORMAT, String], + }) + respondWith!: string; - @Prop({ - default: CONTENT_FORMAT.CONTENT, - type: [CONTENT_FORMAT, String] - }) - respondWith!: string; + @Prop({ + default: false, + }) + withGeneratedAlt!: boolean; - @Prop({ - default: false, - }) - withGeneratedAlt!: boolean; + @Prop({ default: "all", type: IMAGE_RETENTION_MODE_VALUES }) + retainImages?: (typeof IMAGE_RETENTION_MODES)[number]; - @Prop({ default: 'all', type: IMAGE_RETENTION_MODE_VALUES }) - retainImages?: typeof IMAGE_RETENTION_MODES[number]; + @Prop({ + default: false, + }) + withLinksSummary!: boolean | string; - @Prop({ - default: false, - }) - withLinksSummary!: boolean | string; + @Prop({ + default: false, + }) + withImagesSummary!: boolean; - @Prop({ - default: false, - }) - withImagesSummary!: boolean; + @Prop({ + default: false, + }) + noCache!: boolean; - @Prop({ - default: false, - }) - noCache!: boolean; + @Prop({ + default: false, + }) + noGfm!: string | boolean; - @Prop({ - default: false, - }) - noGfm!: string | boolean; + @Prop() + cacheTolerance?: number; - @Prop() - cacheTolerance?: number; + @Prop({ arrayOf: String }) + targetSelector?: string | string[]; - @Prop({ arrayOf: String }) - targetSelector?: string | string[]; + @Prop({ arrayOf: String }) + waitForSelector?: string | string[]; - @Prop({ arrayOf: String }) - waitForSelector?: string | string[]; + @Prop({ arrayOf: String }) + removeSelector?: string | string[]; - @Prop({ arrayOf: String }) - removeSelector?: string | string[]; + @Prop({ + default: false, + }) + keepImgDataUrl!: boolean; - @Prop({ - default: false, - }) - keepImgDataUrl!: boolean; + @Prop({ + default: false, + type: [String, Boolean], + }) + withIframe!: boolean | "quoted"; - @Prop({ - default: false, - type: [String, Boolean] - }) - withIframe!: boolean | 'quoted'; + @Prop({ + default: false, + }) + withShadowDom!: boolean; - @Prop({ - default: false, - }) - withShadowDom!: boolean; + @Prop({ + arrayOf: String, + }) + setCookies?: Cookie[]; - @Prop({ - arrayOf: String, - }) - setCookies?: Cookie[]; + @Prop() + proxyUrl?: string; - @Prop() - proxyUrl?: string; + @Prop() + proxy?: string; - @Prop() - proxy?: string; + @Prop() + userAgent?: string; - @Prop() - userAgent?: string; + @Prop() + engine?: string; - @Prop() - engine?: string; + @Prop({ + arrayOf: String, + }) + injectPageScript?: string[]; - @Prop({ - arrayOf: String, - }) - injectPageScript?: string[]; + @Prop({ + arrayOf: String, + }) + injectFrameScript?: string[]; - @Prop({ - arrayOf: String, - }) - injectFrameScript?: string[]; + @Prop({ + validate: (v: number) => v > 0 && v <= 180, + type: Number, + nullable: true, + }) + timeout?: number | null; - @Prop({ - validate: (v: number) => v > 0 && v <= 180, - type: Number, - nullable: true, - }) - timeout?: number | null; + @Prop() + locale?: string; - @Prop() - locale?: string; + @Prop() + referer?: string; - @Prop() - referer?: string; + @Prop() + tokenBudget?: number; - @Prop() - tokenBudget?: number; + @Prop() + viewport?: Viewport; - @Prop() - viewport?: Viewport; + @Prop() + instruction?: string; - @Prop() - instruction?: string; + @Prop() + jsonSchema?: object; - @Prop() - jsonSchema?: object; + @Prop() + robotsTxt?: string; - @Prop() - robotsTxt?: string; + @Prop() + doNotTrack?: number | null; - @Prop() - doNotTrack?: number | null; + @Prop() + markdown?: TurnDownTweakableOptions; - @Prop() - markdown?: TurnDownTweakableOptions; + @Prop({ + type: RESPOND_TIMING, + }) + respondTiming?: RESPOND_TIMING; - @Prop({ - type: RESPOND_TIMING, - }) - respondTiming?: RESPOND_TIMING; + _hintIps?: string[]; - _hintIps?: string[]; + static override from(input: any) { + const instance = super.from(input) as CrawlerOptions; + const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined; - static override from(input: any) { - const instance = super.from(input) as CrawlerOptions; - const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined; - - const customMode = ctx?.get('x-respond-with') || ctx?.get('x-return-format'); - if (customMode) { - instance.respondWith = customMode; - } - if (instance.respondWith) { - instance.respondWith = instance.respondWith.toLowerCase(); - } - if (instance.respondWith?.includes('lm')) { - if (instance.respondWith.includes('content') || instance.respondWith.includes('markdown')) { - throw new ParamValidationError({ - path: 'respondWith', - message: `LM formats conflicts with content/markdown.`, - }); - } - } - - const locale = ctx?.get('x-locale'); - if (locale) { - instance.locale = locale; - } - - const referer = ctx?.get('x-referer'); - if (referer) { - instance.referer = referer; - } - - const withGeneratedAlt = ctx?.get('x-with-generated-alt'); - if (withGeneratedAlt) { - instance.withGeneratedAlt = Boolean(withGeneratedAlt); - } - const withLinksSummary = ctx?.get('x-with-links-summary'); - if (withLinksSummary) { - if (withLinksSummary === 'all') { - instance.withLinksSummary = withLinksSummary; - } else { - instance.withLinksSummary = Boolean(withLinksSummary); - } - } - const withImagesSummary = ctx?.get('x-with-images-summary'); - if (withImagesSummary) { - instance.withImagesSummary = Boolean(withImagesSummary); - } - const retainImages = ctx?.get('x-retain-images'); - if (retainImages && IMAGE_RETENTION_MODE_VALUES.has(retainImages)) { - instance.retainImages = retainImages as any; - } - if (instance.withGeneratedAlt) { - instance.retainImages = 'all_p'; - } - const noCache = ctx?.get('x-no-cache'); - if (noCache) { - instance.noCache = Boolean(noCache); - } - if (instance.noCache && instance.cacheTolerance === undefined) { - instance.cacheTolerance = 0; - } - let cacheTolerance = parseInt(ctx?.get('x-cache-tolerance') || ''); - if (!isNaN(cacheTolerance)) { - instance.cacheTolerance = cacheTolerance; - } - - const noGfm = ctx?.get('x-no-gfm'); - if (noGfm) { - instance.noGfm = noGfm === 'table' ? noGfm : Boolean(noGfm); - } - - let timeoutSeconds = parseInt(ctx?.get('x-timeout') || ''); - if (!isNaN(timeoutSeconds) && timeoutSeconds > 0) { - instance.timeout = timeoutSeconds <= 180 ? timeoutSeconds : 180; - } else if (ctx?.get('x-timeout')) { - instance.timeout = null; - } - - const removeSelector = ctx?.get('x-remove-selector')?.split(', ').filter(Boolean); - instance.removeSelector ??= removeSelector?.length ? removeSelector : undefined; - const targetSelector = ctx?.get('x-target-selector')?.split(', ').filter(Boolean); - instance.targetSelector ??= targetSelector?.length ? targetSelector : undefined; - const waitForSelector = ctx?.get('x-wait-for-selector')?.split(', ').filter(Boolean); - instance.waitForSelector ??= (waitForSelector?.length ? waitForSelector : undefined) || instance.targetSelector; - const overrideUserAgent = ctx?.get('x-user-agent') || undefined; - instance.userAgent ??= overrideUserAgent; - - const engine = ctx?.get('x-engine'); - if (engine) { - instance.engine = engine; - } - if (instance.engine) { - instance.engine = instance.engine.toLowerCase(); - } - if (instance.engine === 'vlm') { - instance.engine = ENGINE_TYPE.BROWSER; - instance.respondWith = CONTENT_FORMAT.VLM; - } else if (instance.engine === 'xreadlm-v2') { - instance.engine = ENGINE_TYPE.AUTO; - instance.respondWith = CONTENT_FORMAT.XREAD_LM; - } - - const keepImgDataUrl = ctx?.get('x-keep-img-data-url'); - if (keepImgDataUrl) { - instance.keepImgDataUrl = Boolean(keepImgDataUrl); - } - const withIframe = ctx?.get('x-with-iframe'); - if (withIframe) { - instance.withIframe = withIframe.toLowerCase() === 'quoted' ? 'quoted' : Boolean(withIframe); - } - if (instance.withIframe) { - instance.timeout ??= null; - } - const withShadowDom = ctx?.get('x-with-shadow-dom'); - if (withShadowDom) { - instance.withShadowDom = Boolean(withShadowDom); - } - if (instance.withShadowDom) { - instance.timeout ??= null; - } - - const cookies: Cookie[] = []; - const setCookieHeaders = (ctx?.get('x-set-cookie')?.split(', ') || (instance.setCookies as any as string[])).filter(Boolean); - if (Array.isArray(setCookieHeaders)) { - for (const setCookie of setCookieHeaders) { - cookies.push({ - ...parseSetCookieString(setCookie, { decodeValues: true }), - }); - } - } else if (setCookieHeaders && typeof setCookieHeaders === 'string') { - cookies.push({ - ...parseSetCookieString(setCookieHeaders, { decodeValues: true }), - }); - } - instance.setCookies = cookies; - - const proxyUrl = ctx?.get('x-proxy-url'); - instance.proxyUrl ??= proxyUrl || undefined; - const proxy = ctx?.get('x-proxy'); - instance.proxy ??= proxy || undefined; - const robotsTxt = ctx?.get('x-robots-txt'); - instance.robotsTxt ??= robotsTxt || undefined; - - const tokenBudget = ctx?.get('x-token-budget'); - instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined; - - const baseMode = ctx?.get('x-base'); - if (baseMode) { - instance.base = baseMode as any; - } - - const dnt = ctx?.get('dnt'); - instance.doNotTrack ??= (parseInt(dnt || '') || null); - - const respondTiming = ctx?.get('x-respond-timing'); - if (respondTiming) { - instance.respondTiming ??= respondTiming as RESPOND_TIMING; - } - - if (instance.cacheTolerance) { - instance.cacheTolerance = instance.cacheTolerance * 1000; - } - - if (ctx) { - instance.markdown ??= TurnDownTweakableOptions.fromCtx(ctx); - } - - return instance; + const customMode = + ctx?.get("x-respond-with") || ctx?.get("x-return-format"); + if (customMode) { + instance.respondWith = customMode; + } + if (instance.respondWith) { + instance.respondWith = instance.respondWith.toLowerCase(); + } + if (instance.respondWith?.includes("lm")) { + if ( + instance.respondWith.includes("content") || + instance.respondWith.includes("markdown") + ) { + throw new ParamValidationError({ + path: "respondWith", + message: `LM formats conflicts with content/markdown.`, + }); + } } - get presumedRespondTiming() { - if (this.respondTiming) { - return this.respondTiming; - } - if (this.timeout && this.timeout >= 20) { - return RESPOND_TIMING.NETWORK_IDLE; - } - if (this.respondWith.includes('shot') || this.respondWith.includes('vlm')) { - return RESPOND_TIMING.MEDIA_IDLE; - } - - return RESPOND_TIMING.RESOURCE_IDLE; + const locale = ctx?.get("x-locale"); + if (locale) { + instance.locale = locale; } - isSnapshotAcceptableForEarlyResponse(snapshot: PageSnapshot) { - if (this.waitForSelector?.length) { - return false; - } - const presumedTiming = this.presumedRespondTiming; - if (presumedTiming === RESPOND_TIMING.MEDIA_IDLE && snapshot.lastMediaResourceLoaded && snapshot.lastMutationIdle) { - const now = Date.now(); - if ((Math.max(snapshot.lastMediaResourceLoaded, snapshot.lastContentResourceLoaded || 0) + 500) < now) { - return true; - } - } - if ((this.respondWith.includes('vlm') || this.respondWith.includes('pageshot')) && !snapshot.pageshot) { - return false; - } - if ((this.respondWith.includes('vlm') || this.respondWith.includes('screenshot')) && !snapshot.screenshot) { - return false; - } - if (presumedTiming === RESPOND_TIMING.RESOURCE_IDLE && snapshot.lastContentResourceLoaded && snapshot.lastMutationIdle) { - const now = Date.now(); - if ((snapshot.lastContentResourceLoaded + 500) < now) { - return true; - } - } - if (this.injectFrameScript?.length || this.injectPageScript?.length) { - return false; - } - if (presumedTiming === RESPOND_TIMING.VISIBLE_CONTENT && snapshot.parsed?.content) { - return true; - } - if (presumedTiming === RESPOND_TIMING.HTML && snapshot.html) { - return true; - } - if (presumedTiming === RESPOND_TIMING.NETWORK_IDLE) { - return false; - } - if (presumedTiming === RESPOND_TIMING.MUTATION_IDLE && snapshot.lastMutationIdle) { - return true; - } - if (this.respondWith.includes('lm')) { - return false; - } - if (this.withIframe) { - return false; - } - - return !snapshot.isIntermediate; + const referer = ctx?.get("x-referer"); + if (referer) { + instance.referer = referer; } - isCacheQueryApplicable() { - if (this.noCache) { - return false; - } - if (this.cacheTolerance === 0) { - return false; - } - if (this.setCookies?.length) { - return false; - } - if (this.injectFrameScript?.length || this.injectPageScript?.length) { - return false; - } - if (this.viewport) { - return false; - } + const withGeneratedAlt = ctx?.get("x-with-generated-alt"); + if (withGeneratedAlt) { + instance.withGeneratedAlt = Boolean(withGeneratedAlt); + } + const withLinksSummary = ctx?.get("x-with-links-summary"); + if (withLinksSummary) { + if (withLinksSummary === "all") { + instance.withLinksSummary = withLinksSummary; + } else { + instance.withLinksSummary = Boolean(withLinksSummary); + } + } + const withImagesSummary = ctx?.get("x-with-images-summary"); + if (withImagesSummary) { + instance.withImagesSummary = Boolean(withImagesSummary); + } + const retainImages = ctx?.get("x-retain-images"); + if (retainImages && IMAGE_RETENTION_MODE_VALUES.has(retainImages)) { + instance.retainImages = retainImages as any; + } + if (instance.withGeneratedAlt) { + instance.retainImages = "all_p"; + } + const noCache = ctx?.get("x-no-cache"); + if (noCache) { + instance.noCache = Boolean(noCache); + } + if (instance.noCache && instance.cacheTolerance === undefined) { + instance.cacheTolerance = 0; + } + let cacheTolerance = parseInt(ctx?.get("x-cache-tolerance") || ""); + if (!isNaN(cacheTolerance)) { + instance.cacheTolerance = cacheTolerance; + } + const noGfm = ctx?.get("x-no-gfm"); + if (noGfm) { + instance.noGfm = noGfm === "table" ? noGfm : Boolean(noGfm); + } + + let timeoutSeconds = parseInt(ctx?.get("x-timeout") || ""); + if (!isNaN(timeoutSeconds) && timeoutSeconds > 0) { + instance.timeout = timeoutSeconds <= 180 ? timeoutSeconds : 180; + } else if (ctx?.get("x-timeout")) { + instance.timeout = null; + } + + const removeSelector = ctx + ?.get("x-remove-selector") + ?.split(", ") + .filter(Boolean); + instance.removeSelector ??= removeSelector?.length + ? removeSelector + : undefined; + const targetSelector = ctx + ?.get("x-target-selector") + ?.split(", ") + .filter(Boolean); + instance.targetSelector ??= targetSelector?.length + ? targetSelector + : undefined; + const waitForSelector = ctx + ?.get("x-wait-for-selector") + ?.split(", ") + .filter(Boolean); + instance.waitForSelector ??= + (waitForSelector?.length ? waitForSelector : undefined) || + instance.targetSelector; + const overrideUserAgent = ctx?.get("x-user-agent") || undefined; + instance.userAgent ??= overrideUserAgent; + + const engine = ctx?.get("x-engine"); + if (engine) { + instance.engine = engine; + } + if (instance.engine) { + instance.engine = instance.engine.toLowerCase(); + } + if (instance.engine === "vlm") { + instance.engine = ENGINE_TYPE.BROWSER; + instance.respondWith = CONTENT_FORMAT.VLM; + } else if (instance.engine === "xreadlm-v2") { + instance.engine = ENGINE_TYPE.AUTO; + instance.respondWith = CONTENT_FORMAT.XREAD_LM; + } + + const keepImgDataUrl = ctx?.get("x-keep-img-data-url"); + if (keepImgDataUrl) { + instance.keepImgDataUrl = Boolean(keepImgDataUrl); + } + const withIframe = ctx?.get("x-with-iframe"); + if (withIframe) { + instance.withIframe = + withIframe.toLowerCase() === "quoted" ? "quoted" : Boolean(withIframe); + } + if (instance.withIframe) { + instance.timeout ??= null; + } + const withShadowDom = ctx?.get("x-with-shadow-dom"); + if (withShadowDom) { + instance.withShadowDom = Boolean(withShadowDom); + } + if (instance.withShadowDom) { + instance.timeout ??= null; + } + + const cookies: Cookie[] = []; + const setCookieHeaders = ( + ctx?.get("x-set-cookie")?.split(", ") || + (instance.setCookies as any as string[]) + ).filter(Boolean); + if (Array.isArray(setCookieHeaders)) { + for (const setCookie of setCookieHeaders) { + cookies.push({ + ...parseSetCookieString(setCookie, { decodeValues: true }), + }); + } + } else if (setCookieHeaders && typeof setCookieHeaders === "string") { + cookies.push({ + ...parseSetCookieString(setCookieHeaders, { decodeValues: true }), + }); + } + instance.setCookies = cookies; + + const proxyUrl = ctx?.get("x-proxy-url"); + instance.proxyUrl ??= proxyUrl || undefined; + const proxy = ctx?.get("x-proxy"); + instance.proxy ??= proxy || undefined; + const robotsTxt = ctx?.get("x-robots-txt"); + instance.robotsTxt ??= robotsTxt || undefined; + + const tokenBudget = ctx?.get("x-token-budget"); + instance.tokenBudget ??= parseInt(tokenBudget || "") || undefined; + + const baseMode = ctx?.get("x-base"); + if (baseMode) { + instance.base = baseMode as any; + } + + const dnt = ctx?.get("dnt"); + instance.doNotTrack ??= parseInt(dnt || "") || null; + + const respondTiming = ctx?.get("x-respond-timing"); + if (respondTiming) { + instance.respondTiming ??= respondTiming as RESPOND_TIMING; + } + + if (instance.cacheTolerance) { + instance.cacheTolerance = instance.cacheTolerance * 1000; + } + + if (ctx) { + instance.markdown ??= TurnDownTweakableOptions.fromCtx(ctx); + } + + return instance; + } + + get presumedRespondTiming() { + if (this.respondTiming) { + return this.respondTiming; + } + if (this.timeout && this.timeout >= 20) { + return RESPOND_TIMING.NETWORK_IDLE; + } + if (this.respondWith.includes("shot") || this.respondWith.includes("vlm")) { + return RESPOND_TIMING.MEDIA_IDLE; + } + + return RESPOND_TIMING.RESOURCE_IDLE; + } + + isSnapshotAcceptableForEarlyResponse(snapshot: PageSnapshot) { + if (this.waitForSelector?.length) { + return false; + } + const presumedTiming = this.presumedRespondTiming; + if ( + presumedTiming === RESPOND_TIMING.MEDIA_IDLE && + snapshot.lastMediaResourceLoaded && + snapshot.lastMutationIdle + ) { + const now = Date.now(); + if ( + Math.max( + snapshot.lastMediaResourceLoaded, + snapshot.lastContentResourceLoaded || 0, + ) + + 500 < + now + ) { return true; + } } - - isRequestingCompoundContentFormat() { - return !CONTENT_FORMAT_VALUES.has(this.respondWith); + if ( + (this.respondWith.includes("vlm") || + this.respondWith.includes("pageshot")) && + !snapshot.pageshot + ) { + return false; } - - browserIsNotRequired() { - if (this.respondTiming && ![RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes(this.respondTiming)) { - return false; - } - if (this.respondWith.includes(CONTENT_FORMAT.PAGESHOT) || this.respondWith.includes(CONTENT_FORMAT.SCREENSHOT)) { - return false; - } - if (this.injectFrameScript?.length || this.injectPageScript?.length) { - return false; - } - if (this.waitForSelector?.length) { - return false; - } - if (this.withIframe || this.withShadowDom) { - return false; - } - if (this.viewport) { - return false; - } - if (this.pdf) { - return false; - } - if (this.html) { - return false; - } - + if ( + (this.respondWith.includes("vlm") || + this.respondWith.includes("screenshot")) && + !snapshot.screenshot + ) { + return false; + } + if ( + presumedTiming === RESPOND_TIMING.RESOURCE_IDLE && + snapshot.lastContentResourceLoaded && + snapshot.lastMutationIdle + ) { + const now = Date.now(); + if (snapshot.lastContentResourceLoaded + 500 < now) { return true; + } } + if (this.injectFrameScript?.length || this.injectPageScript?.length) { + return false; + } + if ( + presumedTiming === RESPOND_TIMING.VISIBLE_CONTENT && + snapshot.parsed?.content + ) { + return true; + } + if (presumedTiming === RESPOND_TIMING.HTML && snapshot.html) { + return true; + } + if (presumedTiming === RESPOND_TIMING.NETWORK_IDLE) { + return false; + } + if ( + presumedTiming === RESPOND_TIMING.MUTATION_IDLE && + snapshot.lastMutationIdle + ) { + return true; + } + if (this.respondWith.includes("lm")) { + return false; + } + if (this.withIframe) { + return false; + } + + return !snapshot.isIntermediate; + } + + isCacheQueryApplicable() { + if (this.noCache) { + return false; + } + if (this.cacheTolerance === 0) { + return false; + } + if (this.setCookies?.length) { + return false; + } + if (this.injectFrameScript?.length || this.injectPageScript?.length) { + return false; + } + if (this.viewport) { + return false; + } + + return true; + } + + isRequestingCompoundContentFormat() { + return !CONTENT_FORMAT_VALUES.has(this.respondWith); + } + + browserIsNotRequired() { + if ( + this.respondTiming && + ![RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes( + this.respondTiming, + ) + ) { + return false; + } + if ( + this.respondWith.includes(CONTENT_FORMAT.PAGESHOT) || + this.respondWith.includes(CONTENT_FORMAT.SCREENSHOT) + ) { + return false; + } + if (this.injectFrameScript?.length || this.injectPageScript?.length) { + return false; + } + if (this.waitForSelector?.length) { + return false; + } + if (this.withIframe || this.withShadowDom) { + return false; + } + if (this.viewport) { + return false; + } + if (this.pdf) { + return false; + } + if (this.html) { + return false; + } + + return true; + } } export class CrawlerOptionsHeaderOnly extends CrawlerOptions { - static override from(input: any) { - const instance = super.from({ - [RPC_CALL_ENVIRONMENT]: Reflect.get(input, RPC_CALL_ENVIRONMENT), - }) as CrawlerOptionsHeaderOnly; + static override from(input: any) { + const instance = super.from({ + [RPC_CALL_ENVIRONMENT]: Reflect.get(input, RPC_CALL_ENVIRONMENT), + }) as CrawlerOptionsHeaderOnly; - return instance; - } + return instance; + } } diff --git a/src/dto/turndown-tweakable-options.ts b/src/dto/turndown-tweakable-options.ts index 7c7e8de..94bf6d0 100644 --- a/src/dto/turndown-tweakable-options.ts +++ b/src/dto/turndown-tweakable-options.ts @@ -1,61 +1,60 @@ -import { AutoCastable, Prop } from 'civkit/civ-rpc'; -import {Context} from '../services/registry'; -import _ from 'lodash'; - +import { AutoCastable, Prop } from "civkit/civ-rpc"; +import { Context } from "../services/registry"; +import _ from "lodash"; export class TurnDownTweakableOptions extends AutoCastable { - @Prop({ - desc: 'Turndown options > headingStyle', - type: new Set(['setext', 'atx']), - }) - headingStyle?: 'setext' | 'atx'; + @Prop({ + desc: "Turndown options > headingStyle", + type: new Set(["setext", "atx"]), + }) + headingStyle?: "setext" | "atx"; - @Prop({ - desc: 'Turndown options > hr', - validate: (v: string) => v.length > 0 && v.length <= 128 - }) - hr?: string; + @Prop({ + desc: "Turndown options > hr", + validate: (v: string) => v.length > 0 && v.length <= 128, + }) + hr?: string; - @Prop({ - desc: 'Turndown options > bulletListMarker', - type: new Set(['-', '+', '*']), - }) - bulletListMarker?: '-' | '+' | '*'; + @Prop({ + desc: "Turndown options > bulletListMarker", + type: new Set(["-", "+", "*"]), + }) + bulletListMarker?: "-" | "+" | "*"; - @Prop({ - desc: 'Turndown options > emDelimiter', - type: new Set(['_', '*']), - }) - emDelimiter?: '_' | '*'; + @Prop({ + desc: "Turndown options > emDelimiter", + type: new Set(["_", "*"]), + }) + emDelimiter?: "_" | "*"; - @Prop({ - desc: 'Turndown options > strongDelimiter', - type: new Set(['__', '**']), - }) - strongDelimiter?: '__' | '**'; + @Prop({ + desc: "Turndown options > strongDelimiter", + type: new Set(["__", "**"]), + }) + strongDelimiter?: "__" | "**"; - @Prop({ - desc: 'Turndown options > linkStyle', - type: new Set(['inlined', 'referenced', 'discarded']), - }) - linkStyle?: 'inlined' | 'referenced' | 'discarded'; + @Prop({ + desc: "Turndown options > linkStyle", + type: new Set(["inlined", "referenced", "discarded"]), + }) + linkStyle?: "inlined" | "referenced" | "discarded"; - @Prop({ - desc: 'Turndown options > linkReferenceStyle', - type: new Set(['full', 'collapsed', 'shortcut', 'discarded']), - }) - linkReferenceStyle?: 'full' | 'collapsed' | 'shortcut' | 'discarded'; + @Prop({ + desc: "Turndown options > linkReferenceStyle", + type: new Set(["full", "collapsed", "shortcut", "discarded"]), + }) + linkReferenceStyle?: "full" | "collapsed" | "shortcut" | "discarded"; - static fromCtx(ctx: Context, prefix= 'x-md-') { - const draft: Record = {}; - for (const [k, v] of Object.entries(ctx.headers)) { - if (k.startsWith(prefix)) { - const prop = k.slice(prefix.length); - const sk = _.camelCase(prop); - draft[sk] = v as string; - } - } - - return this.from(draft); + static fromCtx(ctx: Context, prefix = "x-md-") { + const draft: Record = {}; + for (const [k, v] of Object.entries(ctx.headers)) { + if (k.startsWith(prefix)) { + const prop = k.slice(prefix.length); + const sk = _.camelCase(prop); + draft[sk] = v as string; + } } + + return this.from(draft); + } } diff --git a/src/fetch.d.ts b/src/fetch.d.ts index 2bc68c0..3bd3649 100644 --- a/src/fetch.d.ts +++ b/src/fetch.d.ts @@ -1,13 +1,21 @@ declare global { - export const { - fetch, - FormData, - Headers, - Request, - Response, - File, - }: typeof import('undici'); - export type { FormData, Headers, Request, RequestInit, Response, RequestInit, File } from 'undici'; + export const { + fetch, + FormData, + Headers, + Request, + Response, + File, + }: typeof import("undici"); + export type { + FormData, + Headers, + Request, + RequestInit, + Response, + RequestInit, + File, + } from "undici"; } -export { }; +export {}; diff --git a/src/lib/transform-server-event-stream.ts b/src/lib/transform-server-event-stream.ts index 1f833a7..6cb41d8 100644 --- a/src/lib/transform-server-event-stream.ts +++ b/src/lib/transform-server-event-stream.ts @@ -1,169 +1,195 @@ -import { TPM, parseJSONText } from 'civkit'; -import { Transform, TransformCallback, TransformOptions } from 'stream'; +import { TPM, parseJSONText } from "civkit"; +import { Transform, TransformCallback, TransformOptions } from "stream"; export class InputServerEventStream extends Transform { - cache: string[] = []; + cache: string[] = []; - constructor(options?: TransformOptions) { - super({ - ...options, - readableObjectMode: true - }); + constructor(options?: TransformOptions) { + super({ + ...options, + readableObjectMode: true, + }); + } + + decodeRoutine() { + if (!this.cache.length) { + return; } - decodeRoutine() { - if (!this.cache.length) { - return; - } - - const vecs = this.cache.join('').split(/\r?\n\r?\n/); - this.cache.length = 0; - const lastVec = vecs.pop(); - if (lastVec) { - this.cache.push(lastVec); - } - - for (const x of vecs) { - const lines: string[] = x.split(/\r?\n/); - - const event: { - event?: string; - data?: string; - id?: string; - retry?: number; - } = {}; - - for (const l of lines) { - const columnPos = l.indexOf(':'); - if (columnPos <= 0) { - continue; - } - const key = l.substring(0, columnPos); - const rawValue = l.substring(columnPos + 1); - const value = rawValue.startsWith(' ') ? rawValue.slice(1) : rawValue; - if (key === 'data') { - if (event.data) { - event.data += value || '\n'; - } else if (event.data === '') { - event.data += '\n'; - event.data += value || '\n'; - } else { - event.data = value; - } - } else if (key === 'retry') { - event.retry = parseInt(value, 10); - } else { - Reflect.set(event, key, value); - } - } - - if (event.data) { - const parsed = parseJSONText(event.data); - if (parsed && typeof parsed === 'object') { - event.data = parsed; - } - } - - if (Object.keys(event).length) { - this.push(event); - } - } + const vecs = this.cache.join("").split(/\r?\n\r?\n/); + this.cache.length = 0; + const lastVec = vecs.pop(); + if (lastVec) { + this.cache.push(lastVec); } - override _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { - if (chunk === null) { - this.push(null); + for (const x of vecs) { + const lines: string[] = x.split(/\r?\n/); + + const event: { + event?: string; + data?: string; + id?: string; + retry?: number; + } = {}; + + for (const l of lines) { + const columnPos = l.indexOf(":"); + if (columnPos <= 0) { + continue; } + const key = l.substring(0, columnPos); + const rawValue = l.substring(columnPos + 1); + const value = rawValue.startsWith(" ") ? rawValue.slice(1) : rawValue; + if (key === "data") { + if (event.data) { + event.data += value || "\n"; + } else if (event.data === "") { + event.data += "\n"; + event.data += value || "\n"; + } else { + event.data = value; + } + } else if (key === "retry") { + event.retry = parseInt(value, 10); + } else { + Reflect.set(event, key, value); + } + } - this.cache.push(chunk.toString()); - this.decodeRoutine(); + if (event.data) { + const parsed = parseJSONText(event.data); + if (parsed && typeof parsed === "object") { + event.data = parsed; + } + } - callback(); + if (Object.keys(event).length) { + this.push(event); + } + } + } + + override _transform( + chunk: any, + encoding: BufferEncoding, + callback: TransformCallback, + ): void { + if (chunk === null) { + this.push(null); } - override _final(callback: (error?: Error | null | undefined) => void): void { - this.decodeRoutine(); - callback(); - } + this.cache.push(chunk.toString()); + this.decodeRoutine(); + + callback(); + } + + override _final(callback: (error?: Error | null | undefined) => void): void { + this.decodeRoutine(); + callback(); + } } @TPM({ - contentType: 'text/event-stream', + contentType: "text/event-stream", }) export class OutputServerEventStream extends Transform { - n: number = 0; + n: number = 0; - constructor(options?: TransformOptions) { - super({ - ...options, writableObjectMode: true, encoding: 'utf-8' - }); - } + constructor(options?: TransformOptions) { + super({ + ...options, + writableObjectMode: true, + encoding: "utf-8", + }); + } - encodeRoutine(chunk: { - event?: string; - data?: any; - id?: string; - retry?: number; - } | string) { - if (typeof chunk === 'object') { - const lines: string[] = []; - - if (chunk.event) { - lines.push(`event: ${chunk.event}`); - } - if (chunk.data) { - if (typeof chunk.data === 'string') { - for (const x of chunk.data.split(/\r?\n/)) { - lines.push(`data: ${x}`); - } - } else { - lines.push(`data: ${JSON.stringify(chunk.data)}`); - } - } - if (chunk.id) { - lines.push(`id: ${chunk.id}`); - } - if (chunk.retry) { - lines.push(`retry: ${chunk.retry}`); - } - if (!lines.length) { - lines.push(`data: ${JSON.stringify(chunk)}`); - } - this.push(lines.join('\n')); - this.push('\n\n'); - this.n++; - - return; - } else if (typeof chunk === 'string') { - const lines: string[] = []; - for (const x of chunk.split(/\r?\n/)) { - lines.push(`data: ${x}`); - } - - this.push(lines.join('\n')); - this.push('\n\n'); - this.n++; + encodeRoutine( + chunk: + | { + event?: string; + data?: any; + id?: string; + retry?: number; } - } + | string, + ) { + if (typeof chunk === "object") { + const lines: string[] = []; - override _transform(chunk: any, encoding: BufferEncoding, callback: TransformCallback): void { - if (chunk === null) { - this.push(null); + if (chunk.event) { + lines.push(`event: ${chunk.event}`); + } + if (chunk.data) { + if (typeof chunk.data === "string") { + for (const x of chunk.data.split(/\r?\n/)) { + lines.push(`data: ${x}`); + } + } else { + lines.push(`data: ${JSON.stringify(chunk.data)}`); } + } + if (chunk.id) { + lines.push(`id: ${chunk.id}`); + } + if (chunk.retry) { + lines.push(`retry: ${chunk.retry}`); + } + if (!lines.length) { + lines.push(`data: ${JSON.stringify(chunk)}`); + } + this.push(lines.join("\n")); + this.push("\n\n"); + this.n++; - this.encodeRoutine(chunk); + return; + } else if (typeof chunk === "string") { + const lines: string[] = []; + for (const x of chunk.split(/\r?\n/)) { + lines.push(`data: ${x}`); + } - callback(); + this.push(lines.join("\n")); + this.push("\n\n"); + this.n++; } + } + + override _transform( + chunk: any, + encoding: BufferEncoding, + callback: TransformCallback, + ): void { + if (chunk === null) { + this.push(null); + } + + this.encodeRoutine(chunk); + + callback(); + } } export interface OutputServerEventStream extends Transform { - write(chunk: string | { - event?: string; - data?: any; - id?: string; - retry?: number; - }, callback?: (error: Error | null | undefined) => void): boolean; - write(chunk: any, callback?: (error: Error | null | undefined) => void): boolean; - write(chunk: any, encoding: BufferEncoding, callback?: (error: Error | null | undefined) => void): boolean; + write( + chunk: + | string + | { + event?: string; + data?: any; + id?: string; + retry?: number; + }, + callback?: (error: Error | null | undefined) => void, + ): boolean; + write( + chunk: any, + callback?: (error: Error | null | undefined) => void, + ): boolean; + write( + chunk: any, + encoding: BufferEncoding, + callback?: (error: Error | null | undefined) => void, + ): boolean; } diff --git a/src/services/alt-text.ts b/src/services/alt-text.ts index 071a22d..0c12db8 100644 --- a/src/services/alt-text.ts +++ b/src/services/alt-text.ts @@ -1,134 +1,153 @@ -import { AssertionFailureError, AsyncService, HashManager } from 'civkit'; -import { singleton } from 'tsyringe'; -import { GlobalLogger } from './logger'; -import { CanvasService } from './canvas'; -import { ImageInterrogationManager } from '../shared/services/common-iminterrogate'; -import { ImgBrief } from './puppeteer'; -import { ImgAlt } from '../db/img-alt'; -import { AsyncLocalContext } from './async-context'; +import { AssertionFailureError, AsyncService, HashManager } from "civkit"; +import { singleton } from "tsyringe"; +import { GlobalLogger } from "./logger"; +import { CanvasService } from "./canvas"; +import { ImageInterrogationManager } from "../shared/services/common-iminterrogate"; +import { ImgBrief } from "./puppeteer"; +import { ImgAlt } from "../db/img-alt"; +import { AsyncLocalContext } from "./async-context"; -const md5Hasher = new HashManager('md5', 'hex'); +const md5Hasher = new HashManager("md5", "hex"); @singleton() export class AltTextService extends AsyncService { + altsToIgnore = "image,img,photo,picture,pic,alt,figure,fig".split(","); + logger = this.globalLogger.child({ service: this.constructor.name }); - altsToIgnore = 'image,img,photo,picture,pic,alt,figure,fig'.split(','); - logger = this.globalLogger.child({ service: this.constructor.name }); + constructor( + protected globalLogger: GlobalLogger, + protected imageInterrogator: ImageInterrogationManager, + protected canvasService: CanvasService, + protected asyncLocalContext: AsyncLocalContext, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected imageInterrogator: ImageInterrogationManager, - protected canvasService: CanvasService, - protected asyncLocalContext: AsyncLocalContext + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } + + async caption(url: string) { + try { + const img = await this.canvasService.loadImage(url); + const contentTypeHint = Reflect.get(img, "contentType"); + if (Math.min(img.naturalHeight, img.naturalWidth) <= 1) { + return `A ${img.naturalWidth}x${img.naturalHeight} image, likely be a tacker probe`; + } + if (Math.min(img.naturalHeight, img.naturalWidth) < 64) { + return `A ${img.naturalWidth}x${img.naturalHeight} small image, likely a logo, icon or avatar`; + } + const resized = this.canvasService.fitImageToSquareBox(img, 1024); + const exported = await this.canvasService.canvasToBuffer( + resized, + "image/png", + ); + + const svgHint = contentTypeHint.includes("svg") + ? `Beware this image is a SVG rendered on a gray background, the gray background is not part of the image.\n\n` + : ""; + const svgSystemHint = contentTypeHint.includes("svg") + ? ` Sometimes the system renders SVG on a gray background. When this happens, you must not include the gray background in the description.` + : ""; + + const r = await this.imageInterrogator.interrogate( + "vertex-gemini-2.0-flash", + { + image: exported, + prompt: `${svgHint}Give a concise image caption descriptive sentence in third person. Start directly with the description.`, + system: `You are BLIP2, an image caption model. You will generate Alt Text (in web pages) for any image for a11y purposes. You must not start with "This image is sth...", instead, start direly with "sth..."${svgSystemHint}`, + }, + ); + + return r.replaceAll(/[\n"]|(\.\s*$)/g, "").trim(); + } catch (err) { + throw new AssertionFailureError({ + message: `Could not generate alt text for url ${url}`, + cause: err, + }); + } + } + + async getAltText(imgBrief: ImgBrief) { + if (!imgBrief.src) { + return undefined; + } + if ( + imgBrief.alt && + !this.altsToIgnore.includes(imgBrief.alt.trim().toLowerCase()) ) { - super(...arguments); + return imgBrief.alt; + } + const digest = md5Hasher.hash(imgBrief.src); + const shortDigest = Buffer.from(digest, "hex").toString("base64url"); + let dims: number[] = []; + do { + if (imgBrief.loaded) { + if (imgBrief.naturalWidth && imgBrief.naturalHeight) { + if (Math.min(imgBrief.naturalWidth, imgBrief.naturalHeight) < 64) { + dims = [imgBrief.naturalWidth, imgBrief.naturalHeight]; + break; + } + } + } + + if (imgBrief.width && imgBrief.height) { + if (Math.min(imgBrief.width, imgBrief.height) < 64) { + dims = [imgBrief.width, imgBrief.height]; + break; + } + } + } while (false); + + if (Math.min(...dims) <= 1) { + return `A ${dims[0]}x${dims[1]} image, likely be a tacker probe`; + } + if (Math.min(...dims) < 64) { + return `A ${dims[0]}x${dims[1]} small image, likely a logo, icon or avatar`; } - override async init() { - await this.dependencyReady(); - this.emit('ready'); + const existing = await ImgAlt.fromFirestore(shortDigest); + + if (existing) { + return existing.generatedAlt || existing.originalAlt || ""; } - async caption(url: string) { - try { - const img = await this.canvasService.loadImage(url); - const contentTypeHint = Reflect.get(img, 'contentType'); - if (Math.min(img.naturalHeight, img.naturalWidth) <= 1) { - return `A ${img.naturalWidth}x${img.naturalHeight} image, likely be a tacker probe`; - } - if (Math.min(img.naturalHeight, img.naturalWidth) < 64) { - return `A ${img.naturalWidth}x${img.naturalHeight} small image, likely a logo, icon or avatar`; - } - const resized = this.canvasService.fitImageToSquareBox(img, 1024); - const exported = await this.canvasService.canvasToBuffer(resized, 'image/png'); + let generatedCaption = ""; - const svgHint = contentTypeHint.includes('svg') ? `Beware this image is a SVG rendered on a gray background, the gray background is not part of the image.\n\n` : ''; - const svgSystemHint = contentTypeHint.includes('svg') ? ` Sometimes the system renders SVG on a gray background. When this happens, you must not include the gray background in the description.` : ''; - - const r = await this.imageInterrogator.interrogate('vertex-gemini-2.0-flash', { - image: exported, - prompt: `${svgHint}Give a concise image caption descriptive sentence in third person. Start directly with the description.`, - system: `You are BLIP2, an image caption model. You will generate Alt Text (in web pages) for any image for a11y purposes. You must not start with "This image is sth...", instead, start direly with "sth..."${svgSystemHint}`, - }); - - return r.replaceAll(/[\n"]|(\.\s*$)/g, '').trim(); - } catch (err) { - throw new AssertionFailureError({ message: `Could not generate alt text for url ${url}`, cause: err }); - } + try { + generatedCaption = await this.caption(imgBrief.src); + } catch (err) { + this.logger.warn(`Unable to generate alt text for ${imgBrief.src}`, { + err, + }); } - async getAltText(imgBrief: ImgBrief) { - if (!imgBrief.src) { - return undefined; - } - if (imgBrief.alt && !this.altsToIgnore.includes(imgBrief.alt.trim().toLowerCase())) { - return imgBrief.alt; - } - const digest = md5Hasher.hash(imgBrief.src); - const shortDigest = Buffer.from(digest, 'hex').toString('base64url'); - let dims: number[] = []; - do { - if (imgBrief.loaded) { - if (imgBrief.naturalWidth && imgBrief.naturalHeight) { - if (Math.min(imgBrief.naturalWidth, imgBrief.naturalHeight) < 64) { - dims = [imgBrief.naturalWidth, imgBrief.naturalHeight]; - break; - } - } - } - - if (imgBrief.width && imgBrief.height) { - if (Math.min(imgBrief.width, imgBrief.height) < 64) { - dims = [imgBrief.width, imgBrief.height]; - break; - } - } - - } while (false); - - if (Math.min(...dims) <= 1) { - return `A ${dims[0]}x${dims[1]} image, likely be a tacker probe`; - } - if (Math.min(...dims) < 64) { - return `A ${dims[0]}x${dims[1]} small image, likely a logo, icon or avatar`; - } - - const existing = await ImgAlt.fromFirestore(shortDigest); - - if (existing) { - return existing.generatedAlt || existing.originalAlt || ''; - } - - let generatedCaption = ''; - - try { - generatedCaption = await this.caption(imgBrief.src); - } catch (err) { - this.logger.warn(`Unable to generate alt text for ${imgBrief.src}`, { err }); - } - - if (this.asyncLocalContext.ctx.DNT) { - // Don't cache alt text if DNT is set - return generatedCaption; - } - - // Don't try again until the next day - const expireMixin = generatedCaption ? {} : { expireAt: new Date(Date.now() + 1000 * 3600 * 24) }; - - await ImgAlt.COLLECTION.doc(shortDigest).set( - { - _id: shortDigest, - src: imgBrief.src || '', - width: imgBrief.naturalWidth || 0, - height: imgBrief.naturalHeight || 0, - urlDigest: digest, - originalAlt: imgBrief.alt || '', - generatedAlt: generatedCaption || '', - createdAt: new Date(), - ...expireMixin - }, { merge: true } - ); - - return generatedCaption; + if (this.asyncLocalContext.ctx.DNT) { + // Don't cache alt text if DNT is set + return generatedCaption; } -}; + + // Don't try again until the next day + const expireMixin = generatedCaption + ? {} + : { expireAt: new Date(Date.now() + 1000 * 3600 * 24) }; + + await ImgAlt.COLLECTION.doc(shortDigest).set( + { + _id: shortDigest, + src: imgBrief.src || "", + width: imgBrief.naturalWidth || 0, + height: imgBrief.naturalHeight || 0, + urlDigest: digest, + originalAlt: imgBrief.alt || "", + generatedAlt: generatedCaption || "", + createdAt: new Date(), + ...expireMixin, + }, + { merge: true }, + ); + + return generatedCaption; + } +} diff --git a/src/services/async-context.ts b/src/services/async-context.ts index f99e506..74d2e16 100644 --- a/src/services/async-context.ts +++ b/src/services/async-context.ts @@ -1,10 +1,10 @@ -import { GlobalAsyncContext } from 'civkit/async-context'; -import { container, singleton } from 'tsyringe'; +import { GlobalAsyncContext } from "civkit/async-context"; +import { container, singleton } from "tsyringe"; @singleton() -export class AsyncLocalContext extends GlobalAsyncContext { } +export class AsyncLocalContext extends GlobalAsyncContext {} const instance = container.resolve(AsyncLocalContext); -Reflect.set(process, 'asyncLocalContext', instance); +Reflect.set(process, "asyncLocalContext", instance); export default instance; diff --git a/src/services/blackhole-detector.ts b/src/services/blackhole-detector.ts index abe1070..058373b 100644 --- a/src/services/blackhole-detector.ts +++ b/src/services/blackhole-detector.ts @@ -1,78 +1,86 @@ -import { singleton } from 'tsyringe'; -import { AsyncService } from 'civkit/async-service'; -import { GlobalLogger } from './logger'; -import { delay } from 'civkit/timeout'; - +import { singleton } from "tsyringe"; +import { AsyncService } from "civkit/async-service"; +import { GlobalLogger } from "./logger"; +import { delay } from "civkit/timeout"; @singleton() export class BlackHoleDetector extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); + lastWorkedTs?: number; + lastDoneRequestTs?: number; + lastIncomingRequestTs?: number; - logger = this.globalLogger.child({ service: this.constructor.name }); - lastWorkedTs?: number; - lastDoneRequestTs?: number; - lastIncomingRequestTs?: number; + maxDelay = 1000 * 30; + concurrentRequests = 0; - maxDelay = 1000 * 30; - concurrentRequests = 0; + strikes = 0; - strikes = 0; + constructor(protected globalLogger: GlobalLogger) { + super(...arguments); - constructor(protected globalLogger: GlobalLogger) { - super(...arguments); + if (process.env.NODE_ENV?.startsWith("prod")) { + setInterval(() => { + this.routine(); + }, 1000 * 30).unref(); + } + } - if (process.env.NODE_ENV?.startsWith('prod')) { - setInterval(() => { - this.routine(); - }, 1000 * 30).unref(); - } + override async init() { + await this.dependencyReady(); + this.logger.debug("BlackHoleDetector started"); + this.emit("ready"); + } + + async routine() { + // We give routine a 3s grace period for potentially paused CPU to spin up and process some requests + await delay(3000); + const now = Date.now(); + const lastWorked = this.lastWorkedTs; + if (!lastWorked) { + return; + } + const dt = now - lastWorked; + if ( + this.concurrentRequests > 1 && + this.lastIncomingRequestTs && + lastWorked && + this.lastIncomingRequestTs >= lastWorked && + dt > this.maxDelay * (this.strikes + 1) + ) { + this.logger.warn( + `BlackHole detected, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`, + ); + this.strikes += 1; } - override async init() { - await this.dependencyReady(); - this.logger.debug('BlackHoleDetector started'); - this.emit('ready'); + if (this.strikes >= 3) { + this.logger.error( + `BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`, + ); + process.nextTick(() => { + this.emit( + "error", + new Error( + `BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`, + ), + ); + // process.exit(1); + }); } + } - async routine() { - // We give routine a 3s grace period for potentially paused CPU to spin up and process some requests - await delay(3000); - const now = Date.now(); - const lastWorked = this.lastWorkedTs; - if (!lastWorked) { - return; - } - const dt = (now - lastWorked); - if (this.concurrentRequests > 1 && - this.lastIncomingRequestTs && lastWorked && - this.lastIncomingRequestTs >= lastWorked && - (dt > (this.maxDelay * (this.strikes + 1))) - ) { - this.logger.warn(`BlackHole detected, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`); - this.strikes += 1; - } + incomingRequest() { + this.lastIncomingRequestTs = Date.now(); + this.lastWorkedTs ??= Date.now(); + this.concurrentRequests++; + } + doneWithRequest() { + this.concurrentRequests--; + this.lastDoneRequestTs = Date.now(); + } - if (this.strikes >= 3) { - this.logger.error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`); - process.nextTick(() => { - this.emit('error', new Error(`BlackHole detected for ${this.strikes} strikes, last worked: ${Math.ceil(dt / 1000)}s ago, concurrentRequests: ${this.concurrentRequests}`)); - // process.exit(1); - }); - } - } - - incomingRequest() { - this.lastIncomingRequestTs = Date.now(); - this.lastWorkedTs ??= Date.now(); - this.concurrentRequests++; - } - doneWithRequest() { - this.concurrentRequests--; - this.lastDoneRequestTs = Date.now(); - } - - itWorked() { - this.lastWorkedTs = Date.now(); - this.strikes = 0; - } - -}; + itWorked() { + this.lastWorkedTs = Date.now(); + this.strikes = 0; + } +} diff --git a/src/services/brave-search.ts b/src/services/brave-search.ts index 207e0b7..c35bba6 100644 --- a/src/services/brave-search.ts +++ b/src/services/brave-search.ts @@ -1,188 +1,224 @@ -import { AsyncService, AutoCastable, DownstreamServiceFailureError, Prop, RPC_CALL_ENVIRONMENT, delay, marshalErrorLike } from 'civkit'; -import { singleton } from 'tsyringe'; -import { GlobalLogger } from './logger'; -import { SecretExposer } from '../shared/services/secrets'; -import { BraveSearchHTTP, WebSearchQueryParams } from '../shared/3rd-party/brave-search'; -import { GEOIP_SUPPORTED_LANGUAGES, GeoIPService } from './geoip'; -import { AsyncLocalContext } from './async-context'; -import { WebSearchOptionalHeaderOptions } from '../shared/3rd-party/brave-types'; -import { BlackHoleDetector } from './blackhole-detector'; +import { + AsyncService, + AutoCastable, + DownstreamServiceFailureError, + Prop, + RPC_CALL_ENVIRONMENT, + delay, + marshalErrorLike, +} from "civkit"; +import { singleton } from "tsyringe"; +import { GlobalLogger } from "./logger"; +import { SecretExposer } from "../shared/services/secrets"; +import { + BraveSearchHTTP, + WebSearchQueryParams, +} from "../shared/3rd-party/brave-search"; +import { GEOIP_SUPPORTED_LANGUAGES, GeoIPService } from "./geoip"; +import { AsyncLocalContext } from "./async-context"; +import { WebSearchOptionalHeaderOptions } from "../shared/3rd-party/brave-types"; +import { BlackHoleDetector } from "./blackhole-detector"; type HeaderReadableRequest = { - get(name: string): string | undefined; + get(name: string): string | undefined; }; @singleton() export class BraveSearchService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + braveSearchHTTP!: BraveSearchHTTP; - braveSearchHTTP!: BraveSearchHTTP; + constructor( + protected globalLogger: GlobalLogger, + protected secretExposer: SecretExposer, + protected geoipControl: GeoIPService, + protected threadLocal: AsyncLocalContext, + protected blackHoleDetector: BlackHoleDetector, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected secretExposer: SecretExposer, - protected geoipControl: GeoIPService, - protected threadLocal: AsyncLocalContext, - protected blackHoleDetector: BlackHoleDetector, - ) { - super(...arguments); + override async init() { + await this.dependencyReady(); + this.emit("ready"); + + this.braveSearchHTTP = new BraveSearchHTTP( + this.secretExposer.BRAVE_SEARCH_API_KEY, + ); + } + + async webSearch(query: WebSearchQueryParams) { + const ip = this.threadLocal.get("ip"); + const extraHeaders: WebSearchOptionalHeaderOptions = {}; + if (ip) { + const geoip = await this.geoipControl.lookupCity( + ip, + GEOIP_SUPPORTED_LANGUAGES.EN, + ); + + if (geoip?.city) { + extraHeaders["X-Loc-City"] = encodeURIComponent(geoip.city); + } + if (geoip?.country?.code) { + extraHeaders["X-Loc-Country"] = geoip.country.code; + } + if (geoip?.timezone) { + extraHeaders["X-Loc-Timezone"] = geoip.timezone; + } + if (geoip?.coordinates) { + extraHeaders["X-Loc-Lat"] = `${geoip.coordinates[0]}`; + extraHeaders["X-Loc-Long"] = `${geoip.coordinates[1]}`; + } + if (geoip?.subdivisions?.length) { + extraHeaders["X-Loc-State"] = encodeURIComponent( + `${geoip.subdivisions[0].code}`, + ); + extraHeaders["X-Loc-State-Name"] = encodeURIComponent( + `${geoip.subdivisions[0].name}`, + ); + } + } + if (this.threadLocal.get("userAgent")) { + extraHeaders["User-Agent"] = `${this.threadLocal.get("userAgent")}`; } - override async init() { - await this.dependencyReady(); - this.emit('ready'); - - this.braveSearchHTTP = new BraveSearchHTTP(this.secretExposer.BRAVE_SEARCH_API_KEY); + const encoded = { ...query }; + if (encoded.q) { + encoded.q = + Buffer.from(encoded.q).toString("ascii") === encoded.q + ? encoded.q + : encodeURIComponent(encoded.q); } - async webSearch(query: WebSearchQueryParams) { - const ip = this.threadLocal.get('ip'); - const extraHeaders: WebSearchOptionalHeaderOptions = {}; - if (ip) { - const geoip = await this.geoipControl.lookupCity(ip, GEOIP_SUPPORTED_LANGUAGES.EN); + let maxTries = 11; - if (geoip?.city) { - extraHeaders['X-Loc-City'] = encodeURIComponent(geoip.city); - } - if (geoip?.country?.code) { - extraHeaders['X-Loc-Country'] = geoip.country.code; - } - if (geoip?.timezone) { - extraHeaders['X-Loc-Timezone'] = geoip.timezone; - } - if (geoip?.coordinates) { - extraHeaders['X-Loc-Lat'] = `${geoip.coordinates[0]}`; - extraHeaders['X-Loc-Long'] = `${geoip.coordinates[1]}`; - } - if (geoip?.subdivisions?.length) { - extraHeaders['X-Loc-State'] = encodeURIComponent(`${geoip.subdivisions[0].code}`); - extraHeaders['X-Loc-State-Name'] = encodeURIComponent(`${geoip.subdivisions[0].name}`); - } - } - if (this.threadLocal.get('userAgent')) { - extraHeaders['User-Agent'] = `${this.threadLocal.get('userAgent')}`; - } + while (maxTries--) { + try { + const r = await this.braveSearchHTTP.webSearch(encoded, { + headers: extraHeaders as Record, + }); + this.blackHoleDetector.itWorked(); - const encoded = { ...query }; - if (encoded.q) { - encoded.q = (Buffer.from(encoded.q).toString('ascii') === encoded.q) ? encoded.q : encodeURIComponent(encoded.q); - } - - let maxTries = 11; - - while (maxTries--) { - try { - const r = await this.braveSearchHTTP.webSearch(encoded, { headers: extraHeaders as Record }); - this.blackHoleDetector.itWorked(); - - return r.parsed; - } catch (err: any) { - this.logger.error(`Web search failed: ${err?.message}`, { err: marshalErrorLike(err) }); - if (err?.status === 429) { - await delay(500 + 1000 * Math.random()); - continue; - } - - throw new DownstreamServiceFailureError({ message: `Search failed` }); - } + return r.parsed; + } catch (err: any) { + this.logger.error(`Web search failed: ${err?.message}`, { + err: marshalErrorLike(err), + }); + if (err?.status === 429) { + await delay(500 + 1000 * Math.random()); + continue; } throw new DownstreamServiceFailureError({ message: `Search failed` }); + } } + throw new DownstreamServiceFailureError({ message: `Search failed` }); + } } - export class BraveSearchExplicitOperatorsDto extends AutoCastable { - @Prop({ - arrayOf: String, - desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.` - }) - ext?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.`, + }) + ext?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.` - }) - filetype?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.`, + }) + filetype?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages containing the specified term in the body of the page. Example: to find information about the Nvidia GeForce GTX 1080 Ti, making sure the page contains the keywords “founders edition” in the body, type “nvidia 1080 ti inbody:“founders edition””.` - }) - inbody?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages containing the specified term in the body of the page. Example: to find information about the Nvidia GeForce GTX 1080 Ti, making sure the page contains the keywords “founders edition” in the body, type “nvidia 1080 ti inbody:“founders edition””.`, + }) + inbody?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.` - }) - intitle?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.`, + }) + intitle?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns webpages containing the specified term either in the title or in the body of the page. Example: to find pages about the 2024 Oscars containing the keywords “best costume design” in the page, type “oscars 2024 inpage:“best costume design””.` - }) - inpage?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns webpages containing the specified term either in the title or in the body of the page. Example: to find pages about the 2024 Oscars containing the keywords “best costume design” in the page, type “oscars 2024 inpage:“best costume design””.`, + }) + inpage?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.` - }) - lang?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`, + }) + lang?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.` - }) - loc?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`, + }) + loc?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.` - }) - site?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.`, + }) + site?: string | string[]; - addTo(searchTerm: string) { - const chunks = []; - for (const [key, value] of Object.entries(this)) { - if (value) { - const values = Array.isArray(value) ? value : [value]; - const textValue = values.map((v) => `${key}:${v}`).join(' OR '); - if (textValue) { - chunks.push(textValue); - } - } + addTo(searchTerm: string) { + const chunks = []; + for (const [key, value] of Object.entries(this)) { + if (value) { + const values = Array.isArray(value) ? value : [value]; + const textValue = values.map((v) => `${key}:${v}`).join(" OR "); + if (textValue) { + chunks.push(textValue); } - const opPart = chunks.length > 1 ? chunks.map((x) => `(${x})`).join(' AND ') : chunks; + } + } + const opPart = + chunks.length > 1 ? chunks.map((x) => `(${x})`).join(" AND ") : chunks; - if (opPart.length) { - return [searchTerm, opPart].join(' '); - } - - return searchTerm; + if (opPart.length) { + return [searchTerm, opPart].join(" "); } - static override from(input: any) { - const instance = super.from(input) as BraveSearchExplicitOperatorsDto; - const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as { - req: HeaderReadableRequest, - } | undefined; + return searchTerm; + } - const params = ['ext', 'filetype', 'inbody', 'intitle', 'inpage', 'lang', 'loc', 'site']; - - for (const p of params) { - const customValue = ctx?.req.get(`x-${p}`) || ctx?.req.get(`${p}`); - if (!customValue) { - continue; - } - - const filtered = customValue.split(', ').filter(Boolean); - if (filtered.length) { - Reflect.set(instance, p, filtered); - } + static override from(input: any) { + const instance = super.from(input) as BraveSearchExplicitOperatorsDto; + const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as + | { + req: HeaderReadableRequest; } + | undefined; - return instance; + const params = [ + "ext", + "filetype", + "inbody", + "intitle", + "inpage", + "lang", + "loc", + "site", + ]; + + for (const p of params) { + const customValue = ctx?.req.get(`x-${p}`) || ctx?.req.get(`${p}`); + if (!customValue) { + continue; + } + + const filtered = customValue.split(", ").filter(Boolean); + if (filtered.length) { + Reflect.set(instance, p, filtered); + } } + + return instance; + } } diff --git a/src/services/canvas.ts b/src/services/canvas.ts index 1853a0d..3d2fc71 100644 --- a/src/services/canvas.ts +++ b/src/services/canvas.ts @@ -1,213 +1,279 @@ -import { singleton, container } from 'tsyringe'; -import { AsyncService, mimeOf, ParamValidationError, SubmittedDataMalformedError, /* downloadFile */ } from 'civkit'; -import { readFile } from 'fs/promises'; +import { singleton, container } from "tsyringe"; +import { + AsyncService, + mimeOf, + ParamValidationError, + SubmittedDataMalformedError /* downloadFile */, +} from "civkit"; +import { readFile } from "fs/promises"; -import type canvas from '@napi-rs/canvas'; -export type { Canvas, Image } from '@napi-rs/canvas'; +import type canvas from "@napi-rs/canvas"; +export type { Canvas, Image } from "@napi-rs/canvas"; -import { GlobalLogger } from './logger'; -import { TempFileManager } from './temp-file'; +import { GlobalLogger } from "./logger"; +import { TempFileManager } from "./temp-file"; -import { isMainThread } from 'worker_threads'; -import type { svg2png } from 'svg2png-wasm' with { 'resolution-mode': 'import' }; -import path from 'path'; -import { Threaded } from './threaded'; +import { isMainThread } from "worker_threads"; +import type { svg2png } from "svg2png-wasm" with { + "resolution-mode": "import", +}; +import path from "path"; +import { Threaded } from "./threaded"; const downloadFile = async (uri: string) => { - const resp = await fetch(uri); - if (!(resp.ok && resp.body)) { - throw new Error(`Unexpected response ${resp.statusText}`); - } - const contentLength = parseInt(resp.headers.get('content-length') || '0'); - if (contentLength > 1024 * 1024 * 100) { - throw new Error('File too large'); - } - const buff = await resp.arrayBuffer(); + const resp = await fetch(uri); + if (!(resp.ok && resp.body)) { + throw new Error(`Unexpected response ${resp.statusText}`); + } + const contentLength = parseInt(resp.headers.get("content-length") || "0"); + if (contentLength > 1024 * 1024 * 100) { + throw new Error("File too large"); + } + const buff = await resp.arrayBuffer(); - return { buff, contentType: resp.headers.get('content-type') }; + return { buff, contentType: resp.headers.get("content-type") }; }; @singleton() export class CanvasService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); + svg2png!: typeof svg2png; + canvas!: typeof canvas; - logger = this.globalLogger.child({ service: this.constructor.name }); - svg2png!: typeof svg2png; - canvas!: typeof canvas; + constructor( + protected temp: TempFileManager, + protected globalLogger: GlobalLogger, + ) { + super(...arguments); + } - constructor( - protected temp: TempFileManager, - protected globalLogger: GlobalLogger, - ) { - super(...arguments); + protected async loadBundledFont() { + const bundledFontPath = path.resolve( + __dirname, + "../../licensed/SourceHanSansSC-Regular.otf", + ); + + try { + return { + family: "Source Han Sans SC", + data: await readFile(bundledFontPath), + }; + } catch (err: any) { + if (err?.code === "ENOENT") { + this.logger.warn( + `Bundled font asset unavailable, falling back to system fonts`, + { + err: { code: "ENOENT", path: bundledFontPath }, + }, + ); + return { + family: "WenQuanYi Zen Hei", + data: undefined, + }; + } + + throw err; } + } - protected async loadBundledFont() { - const bundledFontPath = path.resolve(__dirname, '../../licensed/SourceHanSansSC-Regular.otf'); + override async init() { + await this.dependencyReady(); + if (!isMainThread) { + const { createSvg2png, initialize } = require("svg2png-wasm"); + const wasmBuff = await readFile( + path.resolve( + path.dirname(require.resolve("svg2png-wasm")), + "../svg2png_wasm_bg.wasm", + ), + ); + const font = await this.loadBundledFont(); + await initialize(wasmBuff); + this.svg2png = createSvg2png({ + fonts: font.data ? [Uint8Array.from(font.data)] : [], + defaultFontFamily: { + serifFamily: font.family, + sansSerifFamily: font.family, + cursiveFamily: font.family, + fantasyFamily: font.family, + monospaceFamily: font.family, + }, + }); + } + this.canvas = require("@napi-rs/canvas"); - try { - return { - family: 'Source Han Sans SC', - data: await readFile(bundledFontPath), - }; - } catch (err: any) { - if (err?.code === 'ENOENT') { - this.logger.warn(`Bundled font asset unavailable, falling back to system fonts`, { - err: { code: 'ENOENT', path: bundledFontPath }, - }); - return { - family: 'WenQuanYi Zen Hei', - data: undefined, - }; - } + this.emit("ready"); + } - throw err; + @Threaded() + async renderSvgToPng(svgContent: string) { + return this.svg2png(svgContent, { backgroundColor: "#D3D3D3" }); + } + + protected async _loadImage(input: string | Buffer) { + let buff; + let contentType; + do { + if (typeof input === "string") { + if (input.startsWith("data:")) { + const firstComma = input.indexOf(","); + const header = input.slice(0, firstComma); + const data = input.slice(firstComma + 1); + const encoding = header.split(";")[1]; + contentType = header.split(";")[0].split(":")[1]; + if (encoding?.startsWith("base64")) { + buff = Buffer.from(data, "base64"); + } else { + buff = Buffer.from(decodeURIComponent(data), "utf-8"); + } + break; } - } - - override async init() { - await this.dependencyReady(); - if (!isMainThread) { - const { createSvg2png, initialize } = require('svg2png-wasm'); - const wasmBuff = await readFile(path.resolve(path.dirname(require.resolve('svg2png-wasm')), '../svg2png_wasm_bg.wasm')); - const font = await this.loadBundledFont(); - await initialize(wasmBuff); - this.svg2png = createSvg2png({ - fonts: font.data ? [Uint8Array.from(font.data)] : [], - defaultFontFamily: { - serifFamily: font.family, - sansSerifFamily: font.family, - cursiveFamily: font.family, - fantasyFamily: font.family, - monospaceFamily: font.family, - } - }); + if (input.startsWith("http")) { + const r = await downloadFile(input); + buff = Buffer.from(r.buff); + contentType = r.contentType; + break; } - this.canvas = require('@napi-rs/canvas'); + } + if (Buffer.isBuffer(input)) { + buff = input; + const mime = await mimeOf(buff); + contentType = `${mime.mediaType}/${mime.subType}`; + break; + } + throw new ParamValidationError("Invalid input"); + } while (false); - this.emit('ready'); + if (!buff) { + throw new ParamValidationError("Invalid input"); } - @Threaded() - async renderSvgToPng(svgContent: string,) { - return this.svg2png(svgContent, { backgroundColor: '#D3D3D3' }); + if (contentType?.includes("svg")) { + buff = await this.renderSvgToPng(buff.toString("utf-8")); } - protected async _loadImage(input: string | Buffer) { - let buff; - let contentType; - do { - if (typeof input === 'string') { - if (input.startsWith('data:')) { - const firstComma = input.indexOf(','); - const header = input.slice(0, firstComma); - const data = input.slice(firstComma + 1); - const encoding = header.split(';')[1]; - contentType = header.split(';')[0].split(':')[1]; - if (encoding?.startsWith('base64')) { - buff = Buffer.from(data, 'base64'); - } else { - buff = Buffer.from(decodeURIComponent(data), 'utf-8'); - } - break; - } - if (input.startsWith('http')) { - const r = await downloadFile(input); - buff = Buffer.from(r.buff); - contentType = r.contentType; - break; - } - } - if (Buffer.isBuffer(input)) { - buff = input; - const mime = await mimeOf(buff); - contentType = `${mime.mediaType}/${mime.subType}`; - break; - } - throw new ParamValidationError('Invalid input'); - } while (false); + const img = await this.canvas.loadImage(buff); + Reflect.set(img, "contentType", contentType); - if (!buff) { - throw new ParamValidationError('Invalid input'); - } + return img; + } - if (contentType?.includes('svg')) { - buff = await this.renderSvgToPng(buff.toString('utf-8')); - } + async loadImage(uri: string | Buffer) { + const t0 = Date.now(); + try { + const theImage = await this._loadImage(uri); + const t1 = Date.now(); + this.logger.debug(`Image loaded in ${t1 - t0}ms`); - const img = await this.canvas.loadImage(buff); - Reflect.set(img, 'contentType', contentType); + return theImage; + } catch (err: any) { + if ( + err?.message?.includes("Unsupported image type") || + err?.message?.includes("unsupported") + ) { + this.logger.warn(`Failed to load image ${uri.slice(0, 128)}`, { err }); + throw new SubmittedDataMalformedError( + `Unknown image format for ${uri.slice(0, 128)}`, + ); + } + throw err; + } + } - return img; + fitImageToSquareBox( + image: canvas.Image | canvas.Canvas, + size: number = 1024, + ) { + // this.logger.debug(`Fitting image(${ image.width }x${ image.height }) to ${ size } box`); + // const t0 = Date.now(); + if (image.width <= size && image.height <= size) { + if (image instanceof this.canvas.Canvas) { + return image; + } + const canvasInstance = this.canvas.createCanvas( + image.width, + image.height, + ); + const ctx = canvasInstance.getContext("2d"); + ctx.drawImage( + image, + 0, + 0, + image.width, + image.height, + 0, + 0, + canvasInstance.width, + canvasInstance.height, + ); + // this.logger.debug(`No need to resize, copied to canvas in ${ Date.now() - t0 } ms`); + + return canvasInstance; } - async loadImage(uri: string | Buffer) { - const t0 = Date.now(); - try { - const theImage = await this._loadImage(uri); - const t1 = Date.now(); - this.logger.debug(`Image loaded in ${t1 - t0}ms`); + const aspectRatio = image.width / image.height; - return theImage; - } catch (err: any) { - if (err?.message?.includes('Unsupported image type') || err?.message?.includes('unsupported')) { - this.logger.warn(`Failed to load image ${uri.slice(0, 128)}`, { err }); - throw new SubmittedDataMalformedError(`Unknown image format for ${uri.slice(0, 128)}`); - } - throw err; - } - } + const resizedWidth = Math.round( + aspectRatio > 1 ? size : size * aspectRatio, + ); + const resizedHeight = Math.round( + aspectRatio > 1 ? size / aspectRatio : size, + ); - fitImageToSquareBox(image: canvas.Image | canvas.Canvas, size: number = 1024) { - // this.logger.debug(`Fitting image(${ image.width }x${ image.height }) to ${ size } box`); - // const t0 = Date.now(); - if (image.width <= size && image.height <= size) { - if (image instanceof this.canvas.Canvas) { - return image; - } - const canvasInstance = this.canvas.createCanvas(image.width, image.height); - const ctx = canvasInstance.getContext('2d'); - ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, canvasInstance.width, canvasInstance.height); - // this.logger.debug(`No need to resize, copied to canvas in ${ Date.now() - t0 } ms`); + const canvasInstance = this.canvas.createCanvas( + resizedWidth, + resizedHeight, + ); + const ctx = canvasInstance.getContext("2d"); + ctx.drawImage( + image, + 0, + 0, + image.width, + image.height, + 0, + 0, + resizedWidth, + resizedHeight, + ); + // this.logger.debug(`Resized to ${ resizedWidth }x${ resizedHeight } in ${ Date.now() - t0 } ms`); - return canvasInstance; - } + return canvasInstance; + } - const aspectRatio = image.width / image.height; + corpImage( + image: canvas.Image | canvas.Canvas, + x: number, + y: number, + w: number, + h: number, + ) { + // this.logger.debug(`Cropping image(${ image.width }x${ image.height }) to ${ w }x${ h } at ${ x },${ y } `); + // const t0 = Date.now(); + const canvasInstance = this.canvas.createCanvas(w, h); + const ctx = canvasInstance.getContext("2d"); + ctx.drawImage(image, x, y, w, h, 0, 0, w, h); + // this.logger.debug(`Crop complete in ${ Date.now() - t0 } ms`); - const resizedWidth = Math.round(aspectRatio > 1 ? size : size * aspectRatio); - const resizedHeight = Math.round(aspectRatio > 1 ? size / aspectRatio : size); + return canvasInstance; + } - const canvasInstance = this.canvas.createCanvas(resizedWidth, resizedHeight); - const ctx = canvasInstance.getContext('2d'); - ctx.drawImage(image, 0, 0, image.width, image.height, 0, 0, resizedWidth, resizedHeight); - // this.logger.debug(`Resized to ${ resizedWidth }x${ resizedHeight } in ${ Date.now() - t0 } ms`); - - return canvasInstance; - } - - corpImage(image: canvas.Image | canvas.Canvas, x: number, y: number, w: number, h: number) { - // this.logger.debug(`Cropping image(${ image.width }x${ image.height }) to ${ w }x${ h } at ${ x },${ y } `); - // const t0 = Date.now(); - const canvasInstance = this.canvas.createCanvas(w, h); - const ctx = canvasInstance.getContext('2d'); - ctx.drawImage(image, x, y, w, h, 0, 0, w, h); - // this.logger.debug(`Crop complete in ${ Date.now() - t0 } ms`); - - return canvasInstance; - } - - canvasToDataUrl(canvas: canvas.Canvas, mimeType?: 'image/png' | 'image/jpeg') { - // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`); - // const t0 = Date.now(); - return canvas.toDataURLAsync((mimeType || 'image/png') as 'image/png'); - } - - async canvasToBuffer(canvas: canvas.Canvas, mimeType?: 'image/png' | 'image/jpeg') { - // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`); - // const t0 = Date.now(); - return canvas.toBuffer((mimeType || 'image/png') as 'image/png'); - } + canvasToDataUrl( + canvas: canvas.Canvas, + mimeType?: "image/png" | "image/jpeg", + ) { + // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`); + // const t0 = Date.now(); + return canvas.toDataURLAsync((mimeType || "image/png") as "image/png"); + } + async canvasToBuffer( + canvas: canvas.Canvas, + mimeType?: "image/png" | "image/jpeg", + ) { + // this.logger.debug(`Exporting canvas(${ canvas.width }x${ canvas.height })`); + // const t0 = Date.now(); + return canvas.toBuffer((mimeType || "image/png") as "image/png"); + } } const instance = container.resolve(CanvasService); diff --git a/src/services/cf-browser-rendering.ts b/src/services/cf-browser-rendering.ts index 5fa54a0..0f98273 100644 --- a/src/services/cf-browser-rendering.ts +++ b/src/services/cf-browser-rendering.ts @@ -1,52 +1,53 @@ -import { container, singleton } from 'tsyringe'; -import { AsyncService } from 'civkit/async-service'; -import { SecretExposer } from '../shared/services/secrets'; -import { GlobalLogger } from './logger'; -import { CloudFlareHTTP } from '../shared/3rd-party/cloud-flare'; -import { HTTPServiceError } from 'civkit/http'; -import { ServiceNodeResourceDrainError } from './errors'; +import { container, singleton } from "tsyringe"; +import { AsyncService } from "civkit/async-service"; +import { SecretExposer } from "../shared/services/secrets"; +import { GlobalLogger } from "./logger"; +import { CloudFlareHTTP } from "../shared/3rd-party/cloud-flare"; +import { HTTPServiceError } from "civkit/http"; +import { ServiceNodeResourceDrainError } from "./errors"; @singleton() export class CFBrowserRendering extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); + client!: CloudFlareHTTP; - logger = this.globalLogger.child({ service: this.constructor.name }); - client!: CloudFlareHTTP; + constructor( + protected globalLogger: GlobalLogger, + protected secretExposer: SecretExposer, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected secretExposer: SecretExposer, - ) { - super(...arguments); - } + override async init() { + await this.dependencyReady(); + const [account, key] = this.secretExposer.CLOUD_FLARE_API_KEY?.split(":"); + this.client = new CloudFlareHTTP(account, key); + this.emit("ready"); + } - override async init() { - await this.dependencyReady(); - const [account, key] = this.secretExposer.CLOUD_FLARE_API_KEY?.split(':'); - this.client = new CloudFlareHTTP(account, key); + async fetchContent(url: string) { + try { + const r = await this.client.fetchBrowserRenderedHTML({ url }); - this.emit('ready'); - } + return r.parsed.result; + } catch (err) { + if (err instanceof HTTPServiceError) { + if (err.status === 429) { + // Rate limit exceeded, return empty result + this.logger.warn("Cloudflare browser rendering rate limit exceeded", { + url, + }); - async fetchContent(url: string) { - try { - const r = await this.client.fetchBrowserRenderedHTML({ url }); - - return r.parsed.result; - } catch (err) { - if (err instanceof HTTPServiceError) { - if (err.status === 429) { - // Rate limit exceeded, return empty result - this.logger.warn('Cloudflare browser rendering rate limit exceeded', { url }); - - throw new ServiceNodeResourceDrainError(`Cloudflare browser rendering (our account) is at capacity, please try again later or switch to another engine.`,); - } - } - - throw err; + throw new ServiceNodeResourceDrainError( + `Cloudflare browser rendering (our account) is at capacity, please try again later or switch to another engine.`, + ); } - } + } + throw err; + } + } } const instance = container.resolve(CFBrowserRendering); diff --git a/src/services/curl.ts b/src/services/curl.ts index a7c9ed0..a8b0390 100644 --- a/src/services/curl.ts +++ b/src/services/curl.ts @@ -1,277 +1,325 @@ -import { AsyncService } from 'civkit/async-service'; -import { singleton } from 'tsyringe'; -import { parseString as parseSetCookieString } from 'set-cookie-parser'; +import { AsyncService } from "civkit/async-service"; +import { singleton } from "tsyringe"; +import { parseString as parseSetCookieString } from "set-cookie-parser"; -import { ScrappingOptions } from './puppeteer'; -import { GlobalLogger } from './logger'; -import { AssertionFailureError, FancyFile } from 'civkit'; -import { ServiceBadAttemptError } from './errors'; -import { TempFileManager } from '../services/temp-file'; -import { AsyncLocalContext } from './async-context'; -import { BlackHoleDetector } from './blackhole-detector'; +import { ScrappingOptions } from "./puppeteer"; +import { GlobalLogger } from "./logger"; +import { AssertionFailureError, FancyFile } from "civkit"; +import { ServiceBadAttemptError } from "./errors"; +import { TempFileManager } from "../services/temp-file"; +import { AsyncLocalContext } from "./async-context"; +import { BlackHoleDetector } from "./blackhole-detector"; -type HeaderInfo = Record; +type HeaderInfo = Record< + string, + string | string[] | { code: number; reason?: string } | undefined +>; export interface CURLScrappingOptions extends ScrappingOptions { - method?: string; - body?: string | Buffer; + method?: string; + body?: string | Buffer; } @singleton() export class CurlControl extends AsyncService { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - chromeVersion = '132'; - safariVersion = '537.36'; - platform = 'Linux'; - ua = `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/${this.safariVersion} (KHTML, like Gecko) Chrome/${this.chromeVersion}.0.0.0 Safari/${this.safariVersion}`; + chromeVersion = "132"; + safariVersion = "537.36"; + platform = "Linux"; + ua = `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/${this.safariVersion} (KHTML, like Gecko) Chrome/${this.chromeVersion}.0.0.0 Safari/${this.safariVersion}`; - lifeCycleTrack = new WeakMap(); + lifeCycleTrack = new WeakMap(); - constructor( - protected globalLogger: GlobalLogger, - protected tempFileManager: TempFileManager, - protected asyncLocalContext: AsyncLocalContext, - protected blackHoleDetector: BlackHoleDetector, + constructor( + protected globalLogger: GlobalLogger, + protected tempFileManager: TempFileManager, + protected asyncLocalContext: AsyncLocalContext, + protected blackHoleDetector: BlackHoleDetector, + ) { + super(...arguments); + } + + override async init() { + await this.dependencyReady(); + + if (process.platform === "darwin") { + this.platform = "macOS"; + } else if (process.platform === "win32") { + this.platform = "Windows"; + } + + this.emit("ready"); + } + + impersonateChrome(ua: string) { + this.chromeVersion = ua.match(/Chrome\/(\d+)/)?.[1] || this.chromeVersion; + this.safariVersion = + ua.match(/AppleWebKit\/([\d.]+)/)?.[1] || this.safariVersion; + this.ua = ua; + } + + protected buildHeaders(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { + const headers = new Headers(); + headers.set("User-Agent", crawlOpts?.overrideUserAgent || this.ua); + headers.set( + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + ); + headers.set("Accept-Encoding", "gzip, deflate, br"); + headers.set("Accept-Language", crawlOpts?.locale || "en-US,en;q=0.9"); + + for (const [key, value] of Object.entries(crawlOpts?.extraHeaders || {})) { + if (typeof value !== "undefined") { + headers.set(key, `${value}`); + } + } + + if (crawlOpts?.referer) { + headers.set("Referer", crawlOpts.referer); + } + + if (crawlOpts?.cookies?.length) { + const cookieKv: Record = {}; + for (const cookie of crawlOpts.cookies) { + cookieKv[cookie.name] = cookie.value; + } + for (const cookie of crawlOpts.cookies) { + if (cookie.maxAge && cookie.maxAge < 0) { + delete cookieKv[cookie.name]; + continue; + } + if (cookie.expires && cookie.expires < new Date()) { + delete cookieKv[cookie.name]; + continue; + } + if (cookie.secure && urlToCrawl.protocol !== "https:") { + delete cookieKv[cookie.name]; + continue; + } + if (cookie.domain && !urlToCrawl.hostname.endsWith(cookie.domain)) { + delete cookieKv[cookie.name]; + continue; + } + if (cookie.path && !urlToCrawl.pathname.startsWith(cookie.path)) { + delete cookieKv[cookie.name]; + continue; + } + } + const cookieValue = Object.entries(cookieKv) + .map(([key, value]) => `${key}=${encodeURIComponent(value)}`) + .join("; "); + if (cookieValue) { + headers.set("Cookie", cookieValue); + } + } + + return headers; + } + + protected async responseToFancyFile(response: Response, fileName?: string) { + const ab = await response.arrayBuffer(); + const buffer = Buffer.from(ab); + if (!buffer.length) { + return undefined; + } + + const tmpPath = this.tempFileManager.alloc(); + + return FancyFile.auto( + { + fileBuffer: buffer, + mimeType: + response.headers.get("content-type") || "application/octet-stream", + fileName, + size: buffer.length, + }, + tmpPath, + ); + } + + protected headersToHeaderInfo(response: Response): HeaderInfo { + const headerInfo: HeaderInfo = { + result: { + code: response.status, + reason: response.statusText, + }, + }; + + response.headers.forEach((value, key) => { + headerInfo[key] = value; + headerInfo[ + key.replace(/(^|-)([a-z])/g, (_m, p1, p2) => `${p1}${p2.toUpperCase()}`) + ] = value; + }); + + return headerInfo; + } + + async urlToFile1Shot(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { + const response = await fetch(urlToCrawl, { + method: crawlOpts?.method || (crawlOpts?.body ? "POST" : "GET"), + headers: this.buildHeaders(urlToCrawl, crawlOpts), + body: crawlOpts?.body as any, + redirect: "manual", + signal: AbortSignal.timeout(crawlOpts?.timeoutMs || 30_000), + }).catch((err) => { + throw new AssertionFailureError( + `Failed to access ${urlToCrawl.origin}: ${err}`, + ); + }); + + const headerInfo = this.headersToHeaderInfo(response); + const contentDisposition = + response.headers.get("content-disposition") || ""; + const fileName = + contentDisposition.match(/filename="([^"]+)"/i)?.[1] || + urlToCrawl.pathname.split("/").pop() || + "download.bin"; + const data = + response.status >= 300 && response.status < 400 + ? undefined + : await this.responseToFancyFile(response, fileName); + + return { + statusCode: response.status, + statusText: response.statusText, + data, + headers: [headerInfo], + }; + } + + async urlToFile(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { + let leftRedirection = 6; + let cookieRedirects = 0; + let opts = { ...crawlOpts }; + let nextHopUrl = urlToCrawl; + const fakeHeaderInfos: HeaderInfo[] = []; + + do { + const r = await this.urlToFile1Shot(nextHopUrl, opts); + + if ([301, 302, 303, 307, 308].includes(r.statusCode)) { + fakeHeaderInfos.push(...r.headers); + const headers = r.headers[r.headers.length - 1]; + const location = (headers.Location || headers.location) as + | string + | undefined; + const setCookieHeader = headers["Set-Cookie"] || headers["set-cookie"]; + if (setCookieHeader) { + const cookieAssignments = Array.isArray(setCookieHeader) + ? setCookieHeader + : [setCookieHeader]; + const parsed = cookieAssignments + .filter(Boolean) + .map((x) => parseSetCookieString(`${x}`, { decodeValues: true })); + if (parsed.length) { + opts.cookies = [...(opts.cookies || []), ...parsed]; + } + if (!location) { + cookieRedirects += 1; + } + } + + if (!location && !setCookieHeader) { + return { + statusCode: r.statusCode, + statusText: r.statusText, + data: r.data, + headers: fakeHeaderInfos.concat(r.headers), + }; + } + + if (!location && cookieRedirects > 1) { + throw new ServiceBadAttemptError( + `Failed to access ${urlToCrawl}: Browser required to solve complex cookie preconditions.`, + ); + } + + nextHopUrl = new URL(location || "", nextHopUrl); + leftRedirection -= 1; + continue; + } + + return { + statusCode: r.statusCode, + statusText: r.statusText, + data: r.data, + headers: fakeHeaderInfos.concat(r.headers), + }; + } while (leftRedirection > 0); + + throw new ServiceBadAttemptError( + `Failed to access ${urlToCrawl}: Too many redirections.`, + ); + } + + async sideLoad(targetUrl: URL, crawlOpts?: CURLScrappingOptions) { + const curlResult = await this.urlToFile(targetUrl, crawlOpts); + this.blackHoleDetector.itWorked(); + + let finalURL = targetUrl; + const sideLoadOpts: CURLScrappingOptions["sideLoad"] = { + impersonate: {}, + proxyOrigin: {}, + }; + + for (const headers of curlResult.headers) { + sideLoadOpts.impersonate[finalURL.href] = { + status: (headers.result as any)?.code || -1, + headers: Object.fromEntries( + Object.entries(headers).filter(([key]) => key !== "result"), + ) as Record, + contentType: + `${headers["Content-Type"] || headers["content-type"] || ""}` || + undefined, + }; + if (crawlOpts?.proxyUrl) { + sideLoadOpts.proxyOrigin[finalURL.origin] = crawlOpts.proxyUrl; + } + const code = (headers.result as any)?.code; + if (code && [301, 302, 303, 307, 308].includes(code)) { + const location = (headers.Location || headers.location) as + | string + | undefined; + if (location) { + finalURL = new URL(location, finalURL); + } + } + } + + const lastHeaders = curlResult.headers[curlResult.headers.length - 1]; + const contentType = + `${lastHeaders["Content-Type"] || lastHeaders["content-type"] || ""}`.toLowerCase() || + (await curlResult.data?.mimeType) || + "application/octet-stream"; + const contentDisposition = + `${lastHeaders["Content-Disposition"] || lastHeaders["content-disposition"] || ""}` || + undefined; + const fileName = + contentDisposition?.match(/filename="([^"]+)"/i)?.[1] || + finalURL.pathname.split("/").pop(); + + if ( + sideLoadOpts.impersonate[finalURL.href] && + (await curlResult.data?.size) ) { - super(...arguments); + sideLoadOpts.impersonate[finalURL.href].body = curlResult.data; } - override async init() { - await this.dependencyReady(); + this.lifeCycleTrack.set(this.asyncLocalContext.ctx, curlResult.data); - if (process.platform === 'darwin') { - this.platform = 'macOS'; - } else if (process.platform === 'win32') { - this.platform = 'Windows'; - } - - this.emit('ready'); - } - - impersonateChrome(ua: string) { - this.chromeVersion = ua.match(/Chrome\/(\d+)/)?.[1] || this.chromeVersion; - this.safariVersion = ua.match(/AppleWebKit\/([\d.]+)/)?.[1] || this.safariVersion; - this.ua = ua; - } - - protected buildHeaders(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { - const headers = new Headers(); - headers.set('User-Agent', crawlOpts?.overrideUserAgent || this.ua); - headers.set('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8'); - headers.set('Accept-Encoding', 'gzip, deflate, br'); - headers.set('Accept-Language', crawlOpts?.locale || 'en-US,en;q=0.9'); - - for (const [key, value] of Object.entries(crawlOpts?.extraHeaders || {})) { - if (typeof value !== 'undefined') { - headers.set(key, `${value}`); - } - } - - if (crawlOpts?.referer) { - headers.set('Referer', crawlOpts.referer); - } - - if (crawlOpts?.cookies?.length) { - const cookieKv: Record = {}; - for (const cookie of crawlOpts.cookies) { - cookieKv[cookie.name] = cookie.value; - } - for (const cookie of crawlOpts.cookies) { - if (cookie.maxAge && cookie.maxAge < 0) { - delete cookieKv[cookie.name]; - continue; - } - if (cookie.expires && cookie.expires < new Date()) { - delete cookieKv[cookie.name]; - continue; - } - if (cookie.secure && urlToCrawl.protocol !== 'https:') { - delete cookieKv[cookie.name]; - continue; - } - if (cookie.domain && !urlToCrawl.hostname.endsWith(cookie.domain)) { - delete cookieKv[cookie.name]; - continue; - } - if (cookie.path && !urlToCrawl.pathname.startsWith(cookie.path)) { - delete cookieKv[cookie.name]; - continue; - } - } - const cookieValue = Object.entries(cookieKv).map(([key, value]) => `${key}=${encodeURIComponent(value)}`).join('; '); - if (cookieValue) { - headers.set('Cookie', cookieValue); - } - } - - return headers; - } - - protected async responseToFancyFile(response: Response, fileName?: string) { - const ab = await response.arrayBuffer(); - const buffer = Buffer.from(ab); - if (!buffer.length) { - return undefined; - } - - const tmpPath = this.tempFileManager.alloc(); - - return FancyFile.auto({ - fileBuffer: buffer, - mimeType: response.headers.get('content-type') || 'application/octet-stream', - fileName, - size: buffer.length, - }, tmpPath); - } - - protected headersToHeaderInfo(response: Response): HeaderInfo { - const headerInfo: HeaderInfo = { - result: { - code: response.status, - reason: response.statusText, - }, - }; - - response.headers.forEach((value, key) => { - headerInfo[key] = value; - headerInfo[key.replace(/(^|-)([a-z])/g, (_m, p1, p2) => `${p1}${p2.toUpperCase()}`)] = value; - }); - - return headerInfo; - } - - async urlToFile1Shot(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { - const response = await fetch(urlToCrawl, { - method: crawlOpts?.method || (crawlOpts?.body ? 'POST' : 'GET'), - headers: this.buildHeaders(urlToCrawl, crawlOpts), - body: crawlOpts?.body as any, - redirect: 'manual', - signal: AbortSignal.timeout(crawlOpts?.timeoutMs || 30_000), - }).catch((err) => { - throw new AssertionFailureError(`Failed to access ${urlToCrawl.origin}: ${err}`); - }); - - const headerInfo = this.headersToHeaderInfo(response); - const contentDisposition = response.headers.get('content-disposition') || ''; - const fileName = contentDisposition.match(/filename="([^"]+)"/i)?.[1] || urlToCrawl.pathname.split('/').pop() || 'download.bin'; - const data = response.status >= 300 && response.status < 400 ? undefined : await this.responseToFancyFile(response, fileName); - - return { - statusCode: response.status, - statusText: response.statusText, - data, - headers: [headerInfo], - }; - } - - async urlToFile(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { - let leftRedirection = 6; - let cookieRedirects = 0; - let opts = { ...crawlOpts }; - let nextHopUrl = urlToCrawl; - const fakeHeaderInfos: HeaderInfo[] = []; - - do { - const r = await this.urlToFile1Shot(nextHopUrl, opts); - - if ([301, 302, 303, 307, 308].includes(r.statusCode)) { - fakeHeaderInfos.push(...r.headers); - const headers = r.headers[r.headers.length - 1]; - const location = (headers.Location || headers.location) as string | undefined; - const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie']; - if (setCookieHeader) { - const cookieAssignments = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader]; - const parsed = cookieAssignments.filter(Boolean).map((x) => parseSetCookieString(`${x}`, { decodeValues: true })); - if (parsed.length) { - opts.cookies = [...(opts.cookies || []), ...parsed]; - } - if (!location) { - cookieRedirects += 1; - } - } - - if (!location && !setCookieHeader) { - return { - statusCode: r.statusCode, - statusText: r.statusText, - data: r.data, - headers: fakeHeaderInfos.concat(r.headers), - }; - } - - if (!location && cookieRedirects > 1) { - throw new ServiceBadAttemptError(`Failed to access ${urlToCrawl}: Browser required to solve complex cookie preconditions.`); - } - - nextHopUrl = new URL(location || '', nextHopUrl); - leftRedirection -= 1; - continue; - } - - return { - statusCode: r.statusCode, - statusText: r.statusText, - data: r.data, - headers: fakeHeaderInfos.concat(r.headers), - }; - } while (leftRedirection > 0); - - throw new ServiceBadAttemptError(`Failed to access ${urlToCrawl}: Too many redirections.`); - } - - async sideLoad(targetUrl: URL, crawlOpts?: CURLScrappingOptions) { - const curlResult = await this.urlToFile(targetUrl, crawlOpts); - this.blackHoleDetector.itWorked(); - - let finalURL = targetUrl; - const sideLoadOpts: CURLScrappingOptions['sideLoad'] = { - impersonate: {}, - proxyOrigin: {}, - }; - - for (const headers of curlResult.headers) { - sideLoadOpts.impersonate[finalURL.href] = { - status: (headers.result as any)?.code || -1, - headers: Object.fromEntries( - Object.entries(headers).filter(([key]) => key !== 'result') - ) as Record, - contentType: `${headers['Content-Type'] || headers['content-type'] || ''}` || undefined, - }; - if (crawlOpts?.proxyUrl) { - sideLoadOpts.proxyOrigin[finalURL.origin] = crawlOpts.proxyUrl; - } - const code = (headers.result as any)?.code; - if (code && [301, 302, 303, 307, 308].includes(code)) { - const location = (headers.Location || headers.location) as string | undefined; - if (location) { - finalURL = new URL(location, finalURL); - } - } - } - - const lastHeaders = curlResult.headers[curlResult.headers.length - 1]; - const contentType = `${lastHeaders['Content-Type'] || lastHeaders['content-type'] || ''}`.toLowerCase() || (await curlResult.data?.mimeType) || 'application/octet-stream'; - const contentDisposition = `${lastHeaders['Content-Disposition'] || lastHeaders['content-disposition'] || ''}` || undefined; - const fileName = contentDisposition?.match(/filename="([^"]+)"/i)?.[1] || finalURL.pathname.split('/').pop(); - - if (sideLoadOpts.impersonate[finalURL.href] && (await curlResult.data?.size)) { - sideLoadOpts.impersonate[finalURL.href].body = curlResult.data; - } - - this.lifeCycleTrack.set(this.asyncLocalContext.ctx, curlResult.data); - - return { - finalURL, - sideLoadOpts, - chain: curlResult.headers, - status: curlResult.statusCode, - statusText: curlResult.statusText, - headers: lastHeaders, - contentType, - contentDisposition, - fileName, - file: curlResult.data, - }; - } + return { + finalURL, + sideLoadOpts, + chain: curlResult.headers, + status: curlResult.statusCode, + statusText: curlResult.statusText, + headers: lastHeaders, + contentType, + contentDisposition, + fileName, + file: curlResult.data, + }; + } } diff --git a/src/services/errors.ts b/src/services/errors.ts index 8941ff7..4bfbd61 100644 --- a/src/services/errors.ts +++ b/src/services/errors.ts @@ -1,48 +1,48 @@ -import { ApplicationError, StatusCode } from 'civkit/civ-rpc'; -import _ from 'lodash'; -import dayjs from 'dayjs'; -import utc from 'dayjs/plugin/utc'; +import { ApplicationError, StatusCode } from "civkit/civ-rpc"; +import _ from "lodash"; +import dayjs from "dayjs"; +import utc from "dayjs/plugin/utc"; dayjs.extend(utc); @StatusCode(50301) -export class ServiceDisabledError extends ApplicationError { } +export class ServiceDisabledError extends ApplicationError {} @StatusCode(50302) -export class ServiceCrashedError extends ApplicationError { } +export class ServiceCrashedError extends ApplicationError {} @StatusCode(50303) -export class ServiceNodeResourceDrainError extends ApplicationError { } +export class ServiceNodeResourceDrainError extends ApplicationError {} @StatusCode(50304) -export class ServiceBadAttemptError extends ApplicationError { } +export class ServiceBadAttemptError extends ApplicationError {} @StatusCode(50305) -export class ServiceBadApproachError extends ServiceBadAttemptError { } +export class ServiceBadApproachError extends ServiceBadAttemptError {} @StatusCode(40104) -export class EmailUnverifiedError extends ApplicationError { } +export class EmailUnverifiedError extends ApplicationError {} @StatusCode(40201) -export class InsufficientCreditsError extends ApplicationError { } +export class InsufficientCreditsError extends ApplicationError {} @StatusCode(40202) -export class TierFeatureConstraintError extends ApplicationError { } +export class TierFeatureConstraintError extends ApplicationError {} @StatusCode(40203) -export class InsufficientBalanceError extends ApplicationError { } +export class InsufficientBalanceError extends ApplicationError {} @StatusCode(40903) -export class LockConflictError extends ApplicationError { } +export class LockConflictError extends ApplicationError {} @StatusCode(40904) -export class BudgetExceededError extends ApplicationError { } +export class BudgetExceededError extends ApplicationError {} @StatusCode(45101) -export class HarmfulContentError extends ApplicationError { } +export class HarmfulContentError extends ApplicationError {} @StatusCode(45102) -export class SecurityCompromiseError extends ApplicationError { } +export class SecurityCompromiseError extends ApplicationError {} @StatusCode(41201) -export class BatchSizeTooLargeError extends ApplicationError { } +export class BatchSizeTooLargeError extends ApplicationError {} diff --git a/src/services/finalizer.ts b/src/services/finalizer.ts index 46db29d..0424f79 100644 --- a/src/services/finalizer.ts +++ b/src/services/finalizer.ts @@ -1,33 +1,37 @@ -import { AbstractFinalizerService } from 'civkit/finalizer'; -import { container, singleton } from 'tsyringe'; -import { isMainThread } from 'worker_threads'; -import { GlobalLogger } from './logger'; +import { AbstractFinalizerService } from "civkit/finalizer"; +import { container, singleton } from "tsyringe"; +import { isMainThread } from "worker_threads"; +import { GlobalLogger } from "./logger"; const realProcessExit = process.exit; process.exit = ((code?: number) => { - if (isMainThread) { - return; - } - return realProcessExit(code); + if (isMainThread) { + return; + } + return realProcessExit(code); }) as typeof process.exit; @singleton() export class FinalizerService extends AbstractFinalizerService { + container = container; + logger = this.globalLogger.child({ service: this.constructor.name }); - container = container; - logger = this.globalLogger.child({ service: this.constructor.name }); + override quitProcess(code?: string | number | null | undefined): never { + return realProcessExit(code); + } - override quitProcess(code?: string | number | null | undefined): never { - return realProcessExit(code); - } + constructor(protected globalLogger: GlobalLogger) { + super(...arguments); + } - constructor(protected globalLogger: GlobalLogger) { - super(...arguments); - } - - override onUnhandledRejection(err: unknown, _triggeringPromise: Promise): void { - this.logger.warn(`Unhandled promise rejection in pid ${process.pid}`, { err }); - } + override onUnhandledRejection( + err: unknown, + _triggeringPromise: Promise, + ): void { + this.logger.warn(`Unhandled promise rejection in pid ${process.pid}`, { + err, + }); + } } const instance = container.resolve(FinalizerService); @@ -35,5 +39,5 @@ export const { Finalizer } = instance.decorators(); export default instance; if (isMainThread) { - instance.serviceReady(); + instance.serviceReady(); } diff --git a/src/services/geoip.ts b/src/services/geoip.ts index ad083b9..6e21032 100644 --- a/src/services/geoip.ts +++ b/src/services/geoip.ts @@ -1,145 +1,168 @@ -import { container, singleton } from 'tsyringe'; -import fsp from 'fs/promises'; -import { CityResponse, Reader } from 'maxmind'; -import { AsyncService, AutoCastable, Prop, runOnce } from 'civkit'; -import { GlobalLogger } from './logger'; -import path from 'path'; -import { Threaded } from './threaded'; +import { container, singleton } from "tsyringe"; +import fsp from "fs/promises"; +import { CityResponse, Reader } from "maxmind"; +import { AsyncService, AutoCastable, Prop, runOnce } from "civkit"; +import { GlobalLogger } from "./logger"; +import path from "path"; +import { Threaded } from "./threaded"; export enum GEOIP_SUPPORTED_LANGUAGES { - EN = 'en', - ZH_CN = 'zh-CN', - JA = 'ja', - DE = 'de', - FR = 'fr', - ES = 'es', - PT_BR = 'pt-BR', - RU = 'ru', + EN = "en", + ZH_CN = "zh-CN", + JA = "ja", + DE = "de", + FR = "fr", + ES = "es", + PT_BR = "pt-BR", + RU = "ru", } export class GeoIPInfo extends AutoCastable { - @Prop() - code?: string; + @Prop() + code?: string; - @Prop() - name?: string; + @Prop() + name?: string; } export class GeoIPCountryInfo extends GeoIPInfo { - @Prop() - eu?: boolean; + @Prop() + eu?: boolean; } export class GeoIPCityResponse extends AutoCastable { - @Prop() - continent?: GeoIPInfo; + @Prop() + continent?: GeoIPInfo; - @Prop() - country?: GeoIPCountryInfo; + @Prop() + country?: GeoIPCountryInfo; - @Prop({ - arrayOf: GeoIPInfo - }) - subdivisions?: GeoIPInfo[]; + @Prop({ + arrayOf: GeoIPInfo, + }) + subdivisions?: GeoIPInfo[]; - @Prop() - city?: string; + @Prop() + city?: string; - @Prop({ - arrayOf: Number - }) - coordinates?: [number, number, number]; + @Prop({ + arrayOf: Number, + }) + coordinates?: [number, number, number]; - @Prop() - timezone?: string; + @Prop() + timezone?: string; } @singleton() export class GeoIPService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + mmdbCity!: Reader; + geoIpUnavailable = false; - mmdbCity!: Reader; - geoIpUnavailable = false; + constructor(protected globalLogger: GlobalLogger) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - ) { - super(...arguments); + override async init() { + await this.dependencyReady(); + + this.emit("ready"); + } + + @runOnce() + async _lazyload() { + const mmdpPath = path.resolve( + __dirname, + "..", + "..", + "licensed", + "GeoLite2-City.mmdb", + ); + + let dbBuff: Buffer; + try { + dbBuff = await fsp.readFile(mmdpPath, { flag: "r", encoding: null }); + } catch (error) { + const maybeErr = error; + if ( + typeof maybeErr === "object" && + maybeErr && + "code" in maybeErr && + maybeErr.code === "ENOENT" + ) { + this.geoIpUnavailable = true; + this.logger.warn( + `GeoIP database is missing at ${mmdpPath}; GeoIP lookups are disabled.`, + ); + return; + } + throw error; } + this.mmdbCity = new Reader(dbBuff); - override async init() { - await this.dependencyReady(); + this.logger.info(`Loaded GeoIP database, ${dbBuff.byteLength} bytes`); + } - this.emit('ready'); + @Threaded() + async lookupCity( + ip: string, + lang: GEOIP_SUPPORTED_LANGUAGES = GEOIP_SUPPORTED_LANGUAGES.EN, + ) { + await this._lazyload(); + if (this.geoIpUnavailable || !this.mmdbCity) { + return undefined; } - @runOnce() - async _lazyload() { - const mmdpPath = path.resolve(__dirname, '..', '..', 'licensed', 'GeoLite2-City.mmdb'); + const r = this.mmdbCity.get(ip); - let dbBuff: Buffer; - try { - dbBuff = await fsp.readFile(mmdpPath, { flag: 'r', encoding: null }); - } catch (error) { - const maybeErr = error; - if (typeof maybeErr === 'object' && maybeErr && 'code' in maybeErr && maybeErr.code === 'ENOENT') { - this.geoIpUnavailable = true; - this.logger.warn(`GeoIP database is missing at ${mmdpPath}; GeoIP lookups are disabled.`); - return; - } - throw error; - } - - this.mmdbCity = new Reader(dbBuff); - - this.logger.info(`Loaded GeoIP database, ${dbBuff.byteLength} bytes`); + if (!r) { + return undefined; } + return GeoIPCityResponse.from({ + continent: r.continent + ? { + code: r.continent?.code, + name: r.continent?.names?.[lang] || r.continent?.names?.en, + } + : undefined, + country: r.country + ? { + code: r.country?.iso_code, + name: r.country?.names?.[lang] || r.country?.names.en, + eu: r.country?.is_in_european_union, + } + : undefined, + city: r.city?.names?.[lang] || r.city?.names?.en, + subdivisions: r.subdivisions?.map((x) => ({ + code: x.iso_code, + name: x.names?.[lang] || x.names?.en, + })), + coordinates: r.location + ? [ + r.location.latitude, + r.location.longitude, + r.location.accuracy_radius, + ] + : undefined, + timezone: r.location?.time_zone, + }); + } - @Threaded() - async lookupCity(ip: string, lang: GEOIP_SUPPORTED_LANGUAGES = GEOIP_SUPPORTED_LANGUAGES.EN) { - await this._lazyload(); - if (this.geoIpUnavailable || !this.mmdbCity) { - return undefined; - } - - const r = this.mmdbCity.get(ip); - - if (!r) { - return undefined; - } - - return GeoIPCityResponse.from({ - continent: r.continent ? { - code: r.continent?.code, - name: r.continent?.names?.[lang] || r.continent?.names?.en, - } : undefined, - country: r.country ? { - code: r.country?.iso_code, - name: r.country?.names?.[lang] || r.country?.names.en, - eu: r.country?.is_in_european_union, - } : undefined, - city: r.city?.names?.[lang] || r.city?.names?.en, - subdivisions: r.subdivisions?.map((x) => ({ - code: x.iso_code, - name: x.names?.[lang] || x.names?.en, - })), - coordinates: r.location ? [ - r.location.latitude, r.location.longitude, r.location.accuracy_radius - ] : undefined, - timezone: r.location?.time_zone, - }); - } - - @Threaded() - async lookupCities(ips: string[], lang: GEOIP_SUPPORTED_LANGUAGES = GEOIP_SUPPORTED_LANGUAGES.EN) { - const r = (await Promise.all(ips.map((ip) => this.lookupCity(ip, lang)))).filter(Boolean) as GeoIPCityResponse[]; - - return r; - } + @Threaded() + async lookupCities( + ips: string[], + lang: GEOIP_SUPPORTED_LANGUAGES = GEOIP_SUPPORTED_LANGUAGES.EN, + ) { + const r = ( + await Promise.all(ips.map((ip) => this.lookupCity(ip, lang))) + ).filter(Boolean) as GeoIPCityResponse[]; + return r; + } } const instance = container.resolve(GeoIPService); diff --git a/src/services/jsdom.ts b/src/services/jsdom.ts index 08f5e53..439ec51 100644 --- a/src/services/jsdom.ts +++ b/src/services/jsdom.ts @@ -1,388 +1,483 @@ -import { container, singleton } from 'tsyringe'; -import { GlobalLogger } from './logger'; -import { ExtendedSnapshot, ImgBrief, PageSnapshot } from './puppeteer'; -import { Readability } from '@mozilla/readability'; -import TurndownService from 'turndown'; -import { Threaded } from '../services/threaded'; -import type { ExtraScrappingOptions } from '../api/crawler'; -import { tailwindClasses } from '../utils/tailwind-classes'; -import { countGPTToken } from '../shared/utils/openai'; -import { AsyncService } from 'civkit/async-service'; -import { ApplicationError, AssertionFailureError } from 'civkit/civ-rpc'; +import { container, singleton } from "tsyringe"; +import { GlobalLogger } from "./logger"; +import { ExtendedSnapshot, ImgBrief, PageSnapshot } from "./puppeteer"; +import { Readability } from "@mozilla/readability"; +import TurndownService from "turndown"; +import { Threaded } from "../services/threaded"; +import type { ExtraScrappingOptions } from "../api/crawler"; +import { tailwindClasses } from "../utils/tailwind-classes"; +import { countGPTToken } from "../shared/utils/openai"; +import { AsyncService } from "civkit/async-service"; +import { ApplicationError, AssertionFailureError } from "civkit/civ-rpc"; -const pLinkedom = import('linkedom'); +const pLinkedom = import("linkedom"); @singleton() export class JSDomControl extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + linkedom!: Awaited; - linkedom!: Awaited; + constructor(protected globalLogger: GlobalLogger) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, + override async init() { + await this.dependencyReady(); + this.linkedom = await pLinkedom; + this.emit("ready"); + } + + async narrowSnapshot( + snapshot: PageSnapshot | undefined, + options?: ExtraScrappingOptions, + ) { + if ( + snapshot?.parsed && + !options?.targetSelector && + !options?.removeSelector && + !options?.withIframe && + !options?.withShadowDom ) { - super(...arguments); + return snapshot; + } + if (!snapshot?.html) { + return snapshot; } - override async init() { - await this.dependencyReady(); - this.linkedom = await pLinkedom; - this.emit('ready'); + try { + // SideLoad contains native objects that cannot go through thread boundaries. + return await this.actualNarrowSnapshot(snapshot, { + ...options, + sideLoad: undefined, + }); + } catch (err: any) { + this.logger.warn(`Error narrowing snapshot`, { err }); + if (err instanceof ApplicationError) { + throw err; + } + + throw new AssertionFailureError( + `Failed to process the page: ${err?.message}`, + ); } + } - async narrowSnapshot(snapshot: PageSnapshot | undefined, options?: ExtraScrappingOptions) { - if (snapshot?.parsed && !options?.targetSelector && !options?.removeSelector && !options?.withIframe && !options?.withShadowDom) { - return snapshot; - } - if (!snapshot?.html) { - return snapshot; - } - - try { - // SideLoad contains native objects that cannot go through thread boundaries. - return await this.actualNarrowSnapshot(snapshot, { ...options, sideLoad: undefined }); - } catch (err: any) { - this.logger.warn(`Error narrowing snapshot`, { err }); - if (err instanceof ApplicationError) { - throw err; - } - - throw new AssertionFailureError(`Failed to process the page: ${err?.message}`); - } + @Threaded() + async actualNarrowSnapshot( + snapshot: PageSnapshot, + options?: ExtraScrappingOptions, + ): Promise { + const t0 = Date.now(); + let sourceHTML = snapshot.html; + if (options?.withShadowDom && snapshot.shadowExpanded) { + sourceHTML = snapshot.shadowExpanded; } - - @Threaded() - async actualNarrowSnapshot(snapshot: PageSnapshot, options?: ExtraScrappingOptions): Promise { - const t0 = Date.now(); - let sourceHTML = snapshot.html; - if (options?.withShadowDom && snapshot.shadowExpanded) { - sourceHTML = snapshot.shadowExpanded; - } - let jsdom = this.linkedom.parseHTML(sourceHTML); - if (!jsdom.window.document.documentElement) { - jsdom = this.linkedom.parseHTML(`${sourceHTML}`); - } - const allNodes: Node[] = []; - jsdom.window.document.querySelectorAll('svg').forEach((x) => x.innerHTML = ''); - if (options?.withIframe) { - jsdom.window.document.querySelectorAll('iframe[src],frame[src]').forEach((x) => { - const src = x.getAttribute('src'); - const thisSnapshot = snapshot.childFrames?.find((f) => f.href === src); - if (options?.withIframe === 'quoted') { - const blockquoteElem = jsdom.window.document.createElement('blockquote'); - const preElem = jsdom.window.document.createElement('pre'); - preElem.innerHTML = thisSnapshot?.text || ''; - blockquoteElem.appendChild(preElem); - x.replaceWith(blockquoteElem); - } else if (thisSnapshot?.html) { - x.innerHTML = thisSnapshot.html; - x.querySelectorAll('script, style').forEach((s) => s.remove()); - if (src) { - x.querySelectorAll('[src]').forEach((el) => { - const imgSrc = el.getAttribute('src')!; - if (URL.canParse(imgSrc, src!)) { - el.setAttribute('src', new URL(imgSrc, src!).toString()); - } - }); - x.querySelectorAll('[href]').forEach((el) => { - const linkHref = el.getAttribute('href')!; - if (URL.canParse(linkHref, src!)) { - el.setAttribute('href', new URL(linkHref, src!).toString()); - } - }); - } + let jsdom = this.linkedom.parseHTML(sourceHTML); + if (!jsdom.window.document.documentElement) { + jsdom = this.linkedom.parseHTML( + `${sourceHTML}`, + ); + } + const allNodes: Node[] = []; + jsdom.window.document + .querySelectorAll("svg") + .forEach((x) => (x.innerHTML = "")); + if (options?.withIframe) { + jsdom.window.document + .querySelectorAll("iframe[src],frame[src]") + .forEach((x) => { + const src = x.getAttribute("src"); + const thisSnapshot = snapshot.childFrames?.find( + (f) => f.href === src, + ); + if (options?.withIframe === "quoted") { + const blockquoteElem = + jsdom.window.document.createElement("blockquote"); + const preElem = jsdom.window.document.createElement("pre"); + preElem.innerHTML = thisSnapshot?.text || ""; + blockquoteElem.appendChild(preElem); + x.replaceWith(blockquoteElem); + } else if (thisSnapshot?.html) { + x.innerHTML = thisSnapshot.html; + x.querySelectorAll("script, style").forEach((s) => s.remove()); + if (src) { + x.querySelectorAll("[src]").forEach((el) => { + const imgSrc = el.getAttribute("src")!; + if (URL.canParse(imgSrc, src!)) { + el.setAttribute("src", new URL(imgSrc, src!).toString()); } - }); - } - - if (Array.isArray(options?.removeSelector)) { - for (const rl of options!.removeSelector) { - jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove()); - } - } else if (options?.removeSelector) { - jsdom.window.document.querySelectorAll(options.removeSelector).forEach((x) => x.remove()); - } - - let bewareTargetContentDoesNotExist = false; - if (Array.isArray(options?.targetSelector)) { - bewareTargetContentDoesNotExist = true; - for (const x of options!.targetSelector.map((x) => jsdom.window.document.querySelectorAll(x))) { - x.forEach((el) => { - if (!allNodes.includes(el)) { - allNodes.push(el); - } - }); - } - } else if (options?.targetSelector) { - bewareTargetContentDoesNotExist = true; - jsdom.window.document.querySelectorAll(options.targetSelector).forEach((el) => { - if (!allNodes.includes(el)) { - allNodes.push(el); + }); + x.querySelectorAll("[href]").forEach((el) => { + const linkHref = el.getAttribute("href")!; + if (URL.canParse(linkHref, src!)) { + el.setAttribute("href", new URL(linkHref, src!).toString()); } - }); - } else { - allNodes.push(jsdom.window.document); - } - - if (!allNodes.length) { - - if (bewareTargetContentDoesNotExist) { - return undefined; + }); } - - return snapshot; - } - const textNodes: HTMLElement[] = []; - let rootDoc: Document; - if (allNodes.length === 1 && allNodes[0].nodeName === '#document' && (allNodes[0] as any).documentElement) { - rootDoc = allNodes[0] as any; - if (rootDoc.body?.innerText) { - textNodes.push(rootDoc.body); - } - } else { - rootDoc = this.linkedom.parseHTML('').window.document; - for (const n of allNodes) { - rootDoc.body.appendChild(n); - rootDoc.body.appendChild(rootDoc.createTextNode('\n\n')); - if ((n as HTMLElement).innerText) { - textNodes.push(n as HTMLElement); - } - } - } - const textChunks = textNodes.map((x) => { - const clone = x.cloneNode(true) as HTMLElement; - clone.querySelectorAll('script,style,link,svg').forEach((s) => s.remove()); - - return clone.innerText; + } }); - - let parsed; - try { - parsed = new Readability(rootDoc.cloneNode(true) as any).parse(); - } catch (err: any) { - this.logger.warn(`Failed to parse selected element`, { err }); - } - - const imgSet = new Set(); - const rebuiltImgs: ImgBrief[] = []; - Array.from(rootDoc.querySelectorAll('img[src],img[data-src]')) - .map((x: any) => [x.getAttribute('src'), x.getAttribute('data-src'), x.getAttribute('alt')]) - .forEach(([u1, u2, alt]) => { - let absUrl: string | undefined; - if (u1) { - try { - const u1Txt = new URL(u1, snapshot.rebase || snapshot.href).toString(); - imgSet.add(u1Txt); - absUrl = u1Txt; - } catch (_err) { - // void 0; - } - } - if (u2) { - try { - const u2Txt = new URL(u2, snapshot.rebase || snapshot.href).toString(); - imgSet.add(u2Txt); - absUrl = u2Txt; - } catch (_err) { - // void 0; - } - } - if (absUrl) { - rebuiltImgs.push({ - src: absUrl, - alt - }); - } - }); - - const r = { - ...snapshot, - title: snapshot.title || jsdom.window.document.title, - description: snapshot.description || - (jsdom.window.document.head?.querySelector('meta[name="description"]')?.getAttribute('content') ?? ''), - parsed, - html: rootDoc.documentElement.outerHTML, - text: textChunks.join('\n'), - imgs: (snapshot.imgs || rebuiltImgs)?.filter((x) => imgSet.has(x.src)) || [], - } as PageSnapshot; - - const dt = Date.now() - t0; - if (dt > 1000) { - this.logger.warn(`Performance issue: Narrowing snapshot took ${dt}ms`, { url: snapshot.href, dt }); - } - - return r; } - @Threaded() - async inferSnapshot(snapshot: PageSnapshot) { - const t0 = Date.now(); - const extendedSnapshot = { ...snapshot } as ExtendedSnapshot; - try { - const jsdom = this.linkedom.parseHTML(snapshot.html); - - jsdom.window.document.querySelectorAll('svg').forEach((x) => x.innerHTML = ''); - const links = Array.from(jsdom.window.document.querySelectorAll('a[href]')) - .map((x: any) => [x.textContent.replace(/\s+/g, ' ').trim(), x.getAttribute('href'),]) - .map(([text, href]) => { - if (!href) { - return undefined; - } - try { - const parsed = new URL(href, snapshot.rebase || snapshot.href); - - return [text, parsed.toString()] as const; - } catch (_err) { - return undefined; - } - }) - .filter(Boolean) as [string, string][]; - - extendedSnapshot.links = links; - - const imgs = Array.from(jsdom.window.document.querySelectorAll('img[src],img[data-src]')) - .map((x: any) => { - let linkPreferredSrc = x.getAttribute('src') || ''; - if (linkPreferredSrc.startsWith('data:')) { - const dataSrc = x.getAttribute('data-src') || ''; - if (dataSrc && !dataSrc.startsWith('data:')) { - linkPreferredSrc = dataSrc; - } - } - - return { - src: new URL(linkPreferredSrc, snapshot.rebase || snapshot.href).toString(), - width: parseInt(x.getAttribute('width') || '0'), - height: parseInt(x.getAttribute('height') || '0'), - alt: x.getAttribute('alt') || x.getAttribute('title'), - }; - }); - - extendedSnapshot.imgs = imgs as any; - } catch (_err) { - void 0; - } - - const dt = Date.now() - t0; - if (dt > 1000) { - this.logger.warn(`Performance issue: Inferring snapshot took ${dt}ms`, { url: snapshot.href, dt }); - } - - return extendedSnapshot; + if (Array.isArray(options?.removeSelector)) { + for (const rl of options!.removeSelector) { + jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove()); + } + } else if (options?.removeSelector) { + jsdom.window.document + .querySelectorAll(options.removeSelector) + .forEach((x) => x.remove()); } - cleanRedundantEmptyLines(text: string) { - const lines = text.split(/\r?\n/g); - const mappedFlag = lines.map((line) => Boolean(line.trim())); - - return lines.filter((_line, i) => mappedFlag[i] || mappedFlag[i - 1]).join('\n'); + let bewareTargetContentDoesNotExist = false; + if (Array.isArray(options?.targetSelector)) { + bewareTargetContentDoesNotExist = true; + for (const x of options!.targetSelector.map((x) => + jsdom.window.document.querySelectorAll(x), + )) { + x.forEach((el) => { + if (!allNodes.includes(el)) { + allNodes.push(el); + } + }); + } + } else if (options?.targetSelector) { + bewareTargetContentDoesNotExist = true; + jsdom.window.document + .querySelectorAll(options.targetSelector) + .forEach((el) => { + if (!allNodes.includes(el)) { + allNodes.push(el); + } + }); + } else { + allNodes.push(jsdom.window.document); } - @Threaded() - async cleanHTMLforLMs(sourceHTML: string, ...discardSelectors: string[]): Promise { - const t0 = Date.now(); - let jsdom = this.linkedom.parseHTML(sourceHTML); - if (!jsdom.window.document.documentElement) { - jsdom = this.linkedom.parseHTML(`${sourceHTML}`); + if (!allNodes.length) { + if (bewareTargetContentDoesNotExist) { + return undefined; + } + + return snapshot; + } + const textNodes: HTMLElement[] = []; + let rootDoc: Document; + if ( + allNodes.length === 1 && + allNodes[0].nodeName === "#document" && + (allNodes[0] as any).documentElement + ) { + rootDoc = allNodes[0] as any; + if (rootDoc.body?.innerText) { + textNodes.push(rootDoc.body); + } + } else { + rootDoc = this.linkedom.parseHTML("").window + .document; + for (const n of allNodes) { + rootDoc.body.appendChild(n); + rootDoc.body.appendChild(rootDoc.createTextNode("\n\n")); + if ((n as HTMLElement).innerText) { + textNodes.push(n as HTMLElement); } + } + } + const textChunks = textNodes.map((x) => { + const clone = x.cloneNode(true) as HTMLElement; + clone + .querySelectorAll("script,style,link,svg") + .forEach((s) => s.remove()); - for (const rl of discardSelectors) { - jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove()); - } + return clone.innerText; + }); - jsdom.window.document.querySelectorAll('img[src],img[data-src]').forEach((x) => { - const src = x.getAttribute('src') || x.getAttribute('data-src'); - if (src?.startsWith('data:')) { - x.setAttribute('src', 'blob:opaque'); - } - x.removeAttribute('data-src'); - x.removeAttribute('srcset'); - }); - - jsdom.window.document.querySelectorAll('[class]').forEach((x) => { - const classes = x.getAttribute('class')?.split(/\s+/g) || []; - const newClasses = classes.filter((c) => !tailwindClasses.has(c)); - x.setAttribute('class', newClasses.join(' ')); - }); - jsdom.window.document.querySelectorAll('[style]').forEach((x) => { - const style = x.getAttribute('style')?.toLocaleLowerCase() || ''; - if (style.startsWith('display: none')) { - return; - } - x.removeAttribute('style'); - }); - const treeWalker = jsdom.window.document.createTreeWalker( - jsdom.window.document, // Start from the root document - 0x80 // Only show comment nodes - ); - - let currentNode; - while ((currentNode = treeWalker.nextNode())) { - currentNode.parentNode?.removeChild(currentNode); // Remove each comment node - } - - jsdom.window.document.querySelectorAll('*').forEach((x) => { - const attrs = x.getAttributeNames(); - for (const attr of attrs) { - if (attr.startsWith('data-') || attr.startsWith('aria-')) { - x.removeAttribute(attr); - } - } - }); - - const final = this.cleanRedundantEmptyLines(jsdom.window.document.documentElement.outerHTML); - - const dt = Date.now() - t0; - if (dt > 1000) { - this.logger.warn(`Performance issue: Cleaning HTML for LMs took ${dt}ms`, { dt }); - } - - return final; + let parsed; + try { + parsed = new Readability(rootDoc.cloneNode(true) as any).parse(); + } catch (err: any) { + this.logger.warn(`Failed to parse selected element`, { err }); } - snippetToElement(snippet?: string, _url?: string) { - const parsed = this.linkedom.parseHTML(snippet || ''); + const imgSet = new Set(); + const rebuiltImgs: ImgBrief[] = []; + Array.from(rootDoc.querySelectorAll("img[src],img[data-src]")) + .map((x: any) => [ + x.getAttribute("src"), + x.getAttribute("data-src"), + x.getAttribute("alt"), + ]) + .forEach(([u1, u2, alt]) => { + let absUrl: string | undefined; + if (u1) { + try { + const u1Txt = new URL( + u1, + snapshot.rebase || snapshot.href, + ).toString(); + imgSet.add(u1Txt); + absUrl = u1Txt; + } catch (_err) { + // void 0; + } + } + if (u2) { + try { + const u2Txt = new URL( + u2, + snapshot.rebase || snapshot.href, + ).toString(); + imgSet.add(u2Txt); + absUrl = u2Txt; + } catch (_err) { + // void 0; + } + } + if (absUrl) { + rebuiltImgs.push({ + src: absUrl, + alt, + }); + } + }); - // Hack for turndown gfm table plugin. - parsed.window.document.querySelectorAll('table').forEach((x) => { - Object.defineProperty(x, 'rows', { value: Array.from(x.querySelectorAll('tr')), enumerable: true }); - }); - Object.defineProperty(parsed.window.document.documentElement, 'cloneNode', { - value: function () { return this; }, - }); + const r = { + ...snapshot, + title: snapshot.title || jsdom.window.document.title, + description: + snapshot.description || + (jsdom.window.document.head + ?.querySelector('meta[name="description"]') + ?.getAttribute("content") ?? + ""), + parsed, + html: rootDoc.documentElement.outerHTML, + text: textChunks.join("\n"), + imgs: + (snapshot.imgs || rebuiltImgs)?.filter((x) => imgSet.has(x.src)) || [], + } as PageSnapshot; - return parsed.window.document.documentElement; + const dt = Date.now() - t0; + if (dt > 1000) { + this.logger.warn(`Performance issue: Narrowing snapshot took ${dt}ms`, { + url: snapshot.href, + dt, + }); } - runTurndown(turndownService: TurndownService, html: TurndownService.Node | string) { - const t0 = Date.now(); + return r; + } - try { - return turndownService.turndown(html); - } finally { - const dt = Date.now() - t0; - if (dt > 1000) { - this.logger.warn(`Performance issue: Turndown took ${dt}ms`, { dt }); - } - } - } + @Threaded() + async inferSnapshot(snapshot: PageSnapshot) { + const t0 = Date.now(); + const extendedSnapshot = { ...snapshot } as ExtendedSnapshot; + try { + const jsdom = this.linkedom.parseHTML(snapshot.html); - @Threaded() - async analyzeHTMLTextLite(sourceHTML: string) { - let jsdom = this.linkedom.parseHTML(sourceHTML); - if (!jsdom.window.document.documentElement) { - jsdom = this.linkedom.parseHTML(`${sourceHTML}`); + jsdom.window.document + .querySelectorAll("svg") + .forEach((x) => (x.innerHTML = "")); + const links = Array.from( + jsdom.window.document.querySelectorAll("a[href]"), + ) + .map((x: any) => [ + x.textContent.replace(/\s+/g, " ").trim(), + x.getAttribute("href"), + ]) + .map(([text, href]) => { + if (!href) { + return undefined; + } + try { + const parsed = new URL(href, snapshot.rebase || snapshot.href); + + return [text, parsed.toString()] as const; + } catch (_err) { + return undefined; + } + }) + .filter(Boolean) as [string, string][]; + + extendedSnapshot.links = links; + + const imgs = Array.from( + jsdom.window.document.querySelectorAll("img[src],img[data-src]"), + ).map((x: any) => { + let linkPreferredSrc = x.getAttribute("src") || ""; + if (linkPreferredSrc.startsWith("data:")) { + const dataSrc = x.getAttribute("data-src") || ""; + if (dataSrc && !dataSrc.startsWith("data:")) { + linkPreferredSrc = dataSrc; + } } - jsdom.window.document.querySelectorAll('script,style,link,svg').forEach((s) => s.remove()); - const text = jsdom.window.document.body.innerText || ''; return { - title: jsdom.window.document.title, - text, - tokens: countGPTToken(text.replaceAll(/[\s\r\n\t]+/g, ' ')), + src: new URL( + linkPreferredSrc, + snapshot.rebase || snapshot.href, + ).toString(), + width: parseInt(x.getAttribute("width") || "0"), + height: parseInt(x.getAttribute("height") || "0"), + alt: x.getAttribute("alt") || x.getAttribute("title"), }; + }); + + extendedSnapshot.imgs = imgs as any; + } catch (_err) { + void 0; } + + const dt = Date.now() - t0; + if (dt > 1000) { + this.logger.warn(`Performance issue: Inferring snapshot took ${dt}ms`, { + url: snapshot.href, + dt, + }); + } + + return extendedSnapshot; + } + + cleanRedundantEmptyLines(text: string) { + const lines = text.split(/\r?\n/g); + const mappedFlag = lines.map((line) => Boolean(line.trim())); + + return lines + .filter((_line, i) => mappedFlag[i] || mappedFlag[i - 1]) + .join("\n"); + } + + @Threaded() + async cleanHTMLforLMs( + sourceHTML: string, + ...discardSelectors: string[] + ): Promise { + const t0 = Date.now(); + let jsdom = this.linkedom.parseHTML(sourceHTML); + if (!jsdom.window.document.documentElement) { + jsdom = this.linkedom.parseHTML( + `${sourceHTML}`, + ); + } + + for (const rl of discardSelectors) { + jsdom.window.document.querySelectorAll(rl).forEach((x) => x.remove()); + } + + jsdom.window.document + .querySelectorAll("img[src],img[data-src]") + .forEach((x) => { + const src = x.getAttribute("src") || x.getAttribute("data-src"); + if (src?.startsWith("data:")) { + x.setAttribute("src", "blob:opaque"); + } + x.removeAttribute("data-src"); + x.removeAttribute("srcset"); + }); + + jsdom.window.document.querySelectorAll("[class]").forEach((x) => { + const classes = x.getAttribute("class")?.split(/\s+/g) || []; + const newClasses = classes.filter((c) => !tailwindClasses.has(c)); + x.setAttribute("class", newClasses.join(" ")); + }); + jsdom.window.document.querySelectorAll("[style]").forEach((x) => { + const style = x.getAttribute("style")?.toLocaleLowerCase() || ""; + if (style.startsWith("display: none")) { + return; + } + x.removeAttribute("style"); + }); + const treeWalker = jsdom.window.document.createTreeWalker( + jsdom.window.document, // Start from the root document + 0x80, // Only show comment nodes + ); + + let currentNode; + while ((currentNode = treeWalker.nextNode())) { + currentNode.parentNode?.removeChild(currentNode); // Remove each comment node + } + + jsdom.window.document.querySelectorAll("*").forEach((x) => { + const attrs = x.getAttributeNames(); + for (const attr of attrs) { + if (attr.startsWith("data-") || attr.startsWith("aria-")) { + x.removeAttribute(attr); + } + } + }); + + const final = this.cleanRedundantEmptyLines( + jsdom.window.document.documentElement.outerHTML, + ); + + const dt = Date.now() - t0; + if (dt > 1000) { + this.logger.warn( + `Performance issue: Cleaning HTML for LMs took ${dt}ms`, + { dt }, + ); + } + + return final; + } + + snippetToElement(snippet?: string, _url?: string) { + const parsed = this.linkedom.parseHTML( + snippet || "", + ); + + // Hack for turndown gfm table plugin. + parsed.window.document.querySelectorAll("table").forEach((x) => { + Object.defineProperty(x, "rows", { + value: Array.from(x.querySelectorAll("tr")), + enumerable: true, + }); + }); + Object.defineProperty(parsed.window.document.documentElement, "cloneNode", { + value: function () { + return this; + }, + }); + + return parsed.window.document.documentElement; + } + + runTurndown( + turndownService: TurndownService, + html: TurndownService.Node | string, + ) { + const t0 = Date.now(); + + try { + return turndownService.turndown(html); + } finally { + const dt = Date.now() - t0; + if (dt > 1000) { + this.logger.warn(`Performance issue: Turndown took ${dt}ms`, { dt }); + } + } + } + + @Threaded() + async analyzeHTMLTextLite(sourceHTML: string) { + let jsdom = this.linkedom.parseHTML(sourceHTML); + if (!jsdom.window.document.documentElement) { + jsdom = this.linkedom.parseHTML( + `${sourceHTML}`, + ); + } + jsdom.window.document + .querySelectorAll("script,style,link,svg") + .forEach((s) => s.remove()); + const text = jsdom.window.document.body.innerText || ""; + + return { + title: jsdom.window.document.title, + text, + tokens: countGPTToken(text.replaceAll(/[\s\r\n\t]+/g, " ")), + }; + } } const jsdomControl = container.resolve(JSDomControl); diff --git a/src/services/lm.ts b/src/services/lm.ts index 3ad62a4..59f82bd 100644 --- a/src/services/lm.ts +++ b/src/services/lm.ts @@ -1,152 +1,169 @@ -import { AsyncService } from 'civkit/async-service'; -import { singleton } from 'tsyringe'; +import { AsyncService } from "civkit/async-service"; +import { singleton } from "tsyringe"; -import { PageSnapshot } from './puppeteer'; -import { GlobalLogger } from './logger'; -import _ from 'lodash'; -import { AssertionFailureError } from 'civkit'; -import { LLMManager } from '../shared/services/common-llm'; -import { JSDomControl } from './jsdom'; +import { PageSnapshot } from "./puppeteer"; +import { GlobalLogger } from "./logger"; +import _ from "lodash"; +import { AssertionFailureError } from "civkit"; +import { LLMManager } from "../shared/services/common-llm"; +import { JSDomControl } from "./jsdom"; -const tripleBackTick = '```'; +const tripleBackTick = "```"; @singleton() export class LmControl extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + constructor( + protected globalLogger: GlobalLogger, + protected commonLLM: LLMManager, + protected jsdomControl: JSDomControl, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected commonLLM: LLMManager, - protected jsdomControl: JSDomControl, - ) { - super(...arguments); + override async init() { + await this.dependencyReady(); + + this.emit("ready"); + } + + async *geminiFromBrowserSnapshot( + snapshot?: PageSnapshot & { + pageshotUrl?: string; + }, + ) { + const pageshot = snapshot?.pageshotUrl || snapshot?.pageshot; + + if (!pageshot) { + throw new AssertionFailureError( + "Screenshot of the page is not available", + ); } - override async init() { - await this.dependencyReady(); + const html = await this.jsdomControl.cleanHTMLforLMs( + snapshot.html, + "script,link,style,textarea,select>option,svg", + ); - this.emit('ready'); + const it = this.commonLLM.iterRun("vertex-gemini-1.5-flash-002", { + prompt: [ + `HTML: \n${html}\n\nSCREENSHOT: \n`, + typeof pageshot === "string" ? new URL(pageshot) : pageshot, + `Convert this webpage into a markdown source file that does not contain HTML tags, retaining the page language and visual structures.`, + ], + + options: { + system: + "You are XreadLM-v7, a model that generates Markdown source files only. No HTML, notes and chit-chats allowed", + stream: true, + }, + }); + + const chunks: string[] = []; + for await (const txt of it) { + chunks.push(txt); + const output: PageSnapshot = { + ...snapshot, + parsed: { + ...snapshot?.parsed, + textContent: chunks.join(""), + }, + }; + yield output; } - async* geminiFromBrowserSnapshot(snapshot?: PageSnapshot & { - pageshotUrl?: string, - }) { - const pageshot = snapshot?.pageshotUrl || snapshot?.pageshot; + return; + } - if (!pageshot) { - throw new AssertionFailureError('Screenshot of the page is not available'); - } - - const html = await this.jsdomControl.cleanHTMLforLMs(snapshot.html, 'script,link,style,textarea,select>option,svg'); - - const it = this.commonLLM.iterRun('vertex-gemini-1.5-flash-002', { - prompt: [ - `HTML: \n${html}\n\nSCREENSHOT: \n`, - typeof pageshot === 'string' ? new URL(pageshot) : pageshot, - `Convert this webpage into a markdown source file that does not contain HTML tags, retaining the page language and visual structures.`, - ], - - options: { - system: 'You are XreadLM-v7, a model that generates Markdown source files only. No HTML, notes and chit-chats allowed', - stream: true - } - }); - - const chunks: string[] = []; - for await (const txt of it) { - chunks.push(txt); - const output: PageSnapshot = { - ...snapshot, - parsed: { - ...snapshot?.parsed, - textContent: chunks.join(''), - } - }; - yield output; - } - - return; + async *xreadLMMarkdownFromSnapshot(snapshot?: PageSnapshot) { + if (!snapshot) { + throw new AssertionFailureError("Snapshot of the page is not available"); } - async* xreadLMMarkdownFromSnapshot(snapshot?: PageSnapshot) { - if (!snapshot) { - throw new AssertionFailureError('Snapshot of the page is not available'); - } + const html = await this.jsdomControl.cleanHTMLforLMs( + snapshot.html, + "script,link,style,textarea,select>option,svg", + ); - const html = await this.jsdomControl.cleanHTMLforLMs(snapshot.html, 'script,link,style,textarea,select>option,svg'); + const it = this.commonLLM.iterRun("xreadlm-v2", { + prompt: `Extract the main content from the given HTML and convert it to Markdown format.\n\n${tripleBackTick}html\n${html}\n${tripleBackTick}\n`, - const it = this.commonLLM.iterRun('xreadlm-v2', { - prompt: `Extract the main content from the given HTML and convert it to Markdown format.\n\n${tripleBackTick}html\n${html}\n${tripleBackTick}\n`, + options: { + // system: 'You are an AI assistant developed by VENDOR_NAME', + stream: true, + modelSpecific: { + top_k: 1, + temperature: 0, + repetition_penalty: 1.13, + presence_penalty: 0.25, + frequency_penalty: 0.25, + max_tokens: 8192, + }, + }, + maxTry: 1, + }); - options: { - // system: 'You are an AI assistant developed by VENDOR_NAME', - stream: true, - modelSpecific: { - top_k: 1, - temperature: 0, - repetition_penalty: 1.13, - presence_penalty: 0.25, - frequency_penalty: 0.25, - max_tokens: 8192, - } - }, - maxTry: 1, - }); - - const chunks: string[] = []; - for await (const txt of it) { - chunks.push(txt); - const output: PageSnapshot = { - ...snapshot, - parsed: { - ...snapshot?.parsed, - textContent: chunks.join(''), - } - }; - yield output; - } - - return; + const chunks: string[] = []; + for await (const txt of it) { + chunks.push(txt); + const output: PageSnapshot = { + ...snapshot, + parsed: { + ...snapshot?.parsed, + textContent: chunks.join(""), + }, + }; + yield output; } - async* xreadLMFromSnapshot(schema?: string, instruction: string = 'Infer useful information from the HTML and present it in a structured JSON object.', snapshot?: PageSnapshot) { - if (!snapshot) { - throw new AssertionFailureError('Snapshot of the page is not available'); - } + return; + } - const html = await this.jsdomControl.cleanHTMLforLMs(snapshot.html, 'script,link,style,textarea,select>option,svg'); - - const it = this.commonLLM.iterRun('xreadlm-v2', { - prompt: `${instruction}\n\n${tripleBackTick}html\n${html}\n${tripleBackTick}\n${schema ? `The JSON schema:\n${tripleBackTick}json\n${schema}\n${tripleBackTick}\n` : ''}`, - options: { - // system: 'You are an AI assistant developed by VENDOR_NAME', - stream: true, - modelSpecific: { - top_k: 1, - temperature: 0, - repetition_penalty: 1.13, - presence_penalty: 0.25, - frequency_penalty: 0.25, - max_tokens: 8192, - } - }, - maxTry: 1, - }); - - const chunks: string[] = []; - for await (const txt of it) { - chunks.push(txt); - const output: PageSnapshot = { - ...snapshot, - parsed: { - ...snapshot?.parsed, - textContent: chunks.join(''), - } - }; - yield output; - } - - return; + async *xreadLMFromSnapshot( + schema?: string, + instruction: string = "Infer useful information from the HTML and present it in a structured JSON object.", + snapshot?: PageSnapshot, + ) { + if (!snapshot) { + throw new AssertionFailureError("Snapshot of the page is not available"); } + + const html = await this.jsdomControl.cleanHTMLforLMs( + snapshot.html, + "script,link,style,textarea,select>option,svg", + ); + + const it = this.commonLLM.iterRun("xreadlm-v2", { + prompt: `${instruction}\n\n${tripleBackTick}html\n${html}\n${tripleBackTick}\n${schema ? `The JSON schema:\n${tripleBackTick}json\n${schema}\n${tripleBackTick}\n` : ""}`, + options: { + // system: 'You are an AI assistant developed by VENDOR_NAME', + stream: true, + modelSpecific: { + top_k: 1, + temperature: 0, + repetition_penalty: 1.13, + presence_penalty: 0.25, + frequency_penalty: 0.25, + max_tokens: 8192, + }, + }, + maxTry: 1, + }); + + const chunks: string[] = []; + for await (const txt of it) { + chunks.push(txt); + const output: PageSnapshot = { + ...snapshot, + parsed: { + ...snapshot?.parsed, + textContent: chunks.join(""), + }, + }; + yield output; + } + + return; + } } diff --git a/src/services/logger.ts b/src/services/logger.ts index a930e9b..8e5c5bb 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -1,56 +1,57 @@ -import { AbstractPinoLogger } from 'civkit/pino-logger'; -import { singleton, container } from 'tsyringe'; -import { threadId } from 'node:worker_threads'; -import { getTraceCtx } from 'civkit/async-context'; +import { AbstractPinoLogger } from "civkit/pino-logger"; +import { singleton, container } from "tsyringe"; +import { threadId } from "node:worker_threads"; +import { getTraceCtx } from "civkit/async-context"; - -const levelToSeverityMap: { [k: string]: string | undefined; } = { - trace: 'DEFAULT', - debug: 'DEBUG', - info: 'INFO', - warn: 'WARNING', - error: 'ERROR', - fatal: 'CRITICAL', +const levelToSeverityMap: { [k: string]: string | undefined } = { + trace: "DEFAULT", + debug: "DEBUG", + info: "INFO", + warn: "WARNING", + error: "ERROR", + fatal: "CRITICAL", }; @singleton() export class GlobalLogger extends AbstractPinoLogger { - loggerOptions = { - level: 'debug', - base: { - tid: threadId, - } - }; + loggerOptions = { + level: "debug", + base: { + tid: threadId, + }, + }; - override init(): void { - if (process.env['NODE_ENV']?.startsWith('prod')) { - super.init(process.stdout); - } else { - const PinoPretty = require('pino-pretty').PinoPretty; - super.init(PinoPretty({ - singleLine: true, - colorize: true, - messageFormat(log: any, messageKey: any) { - return `${log['tid'] ? `[${log['tid']}]` : ''}[${log['service'] || 'ROOT'}] ${log[messageKey]}`; - }, - })); - } - - - this.emit('ready'); + override init(): void { + if (process.env["NODE_ENV"]?.startsWith("prod")) { + super.init(process.stdout); + } else { + const PinoPretty = require("pino-pretty").PinoPretty; + super.init( + PinoPretty({ + singleLine: true, + colorize: true, + messageFormat(log: any, messageKey: any) { + return `${log["tid"] ? `[${log["tid"]}]` : ""}[${log["service"] || "ROOT"}] ${log[messageKey]}`; + }, + }), + ); } - override log(...args: any[]) { - const [levelObj, ...rest] = args; - const severity = levelToSeverityMap[levelObj?.level]; - const traceCtx = getTraceCtx(); - const patched: any= { ...levelObj, severity }; - const traceId = traceCtx?.googleTraceId || traceCtx?.traceId; - if (traceId && process.env['GCLOUD_PROJECT']) { - patched['logging.googleapis.com/trace'] = `projects/${process.env['GCLOUD_PROJECT']}/traces/${traceId}`; - } - return super.log(patched, ...rest); + this.emit("ready"); + } + + override log(...args: any[]) { + const [levelObj, ...rest] = args; + const severity = levelToSeverityMap[levelObj?.level]; + const traceCtx = getTraceCtx(); + const patched: any = { ...levelObj, severity }; + const traceId = traceCtx?.googleTraceId || traceCtx?.traceId; + if (traceId && process.env["GCLOUD_PROJECT"]) { + patched["logging.googleapis.com/trace"] = + `projects/${process.env["GCLOUD_PROJECT"]}/traces/${traceId}`; } + return super.log(patched, ...rest); + } } const instance = container.resolve(GlobalLogger); diff --git a/src/services/minimal-stealth.js b/src/services/minimal-stealth.js index 6f75c2f..e877b58 100644 --- a/src/services/minimal-stealth.js +++ b/src/services/minimal-stealth.js @@ -1,599 +1,612 @@ - - export function minimalStealth() { - /** - * A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving traces. - * - * Meant to be passed down in puppeteer and used in the context of the page (everything in here runs in NodeJS as well as a browser). - * - * Note: If for whatever reason you need to use this outside of `puppeteer-extra`: - * Just remove the `module.exports` statement at the very bottom, the rest can be copy pasted into any browser context. - * - * Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities. - * - */ - const utils = {}; + /** + * A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving traces. + * + * Meant to be passed down in puppeteer and used in the context of the page (everything in here runs in NodeJS as well as a browser). + * + * Note: If for whatever reason you need to use this outside of `puppeteer-extra`: + * Just remove the `module.exports` statement at the very bottom, the rest can be copy pasted into any browser context. + * + * Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities. + * + */ + const utils = {}; - // const toStringRedirects = new WeakMap(); + // const toStringRedirects = new WeakMap(); - utils.init = () => { - utils.preloadCache(); - // const handler = { - // apply: function (target, ctx) { - // // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` - // if (ctx === Function.prototype.toString) { - // return utils.makeNativeString('toString'); - // } + utils.init = () => { + utils.preloadCache(); + // const handler = { + // apply: function (target, ctx) { + // // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` + // if (ctx === Function.prototype.toString) { + // return utils.makeNativeString('toString'); + // } - // const originalObj = toStringRedirects.get(ctx); + // const originalObj = toStringRedirects.get(ctx); - // // `toString` targeted at our proxied Object detected - // if (originalObj) { - // const fallback = () => - // originalObj && originalObj.name - // ? utils.makeNativeString(originalObj.name) - // : utils.makeNativeString(ctx.name); + // // `toString` targeted at our proxied Object detected + // if (originalObj) { + // const fallback = () => + // originalObj && originalObj.name + // ? utils.makeNativeString(originalObj.name) + // : utils.makeNativeString(ctx.name); - // // Return the toString representation of our original object if possible - // return originalObj + '' || fallback(); - // } + // // Return the toString representation of our original object if possible + // return originalObj + '' || fallback(); + // } - // if (typeof ctx === 'undefined' || ctx === null) { - // return target.call(ctx); - // } + // if (typeof ctx === 'undefined' || ctx === null) { + // return target.call(ctx); + // } - // // Check if the toString protype of the context is the same as the global prototype, - // // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case - // const hasSameProto = Object.getPrototypeOf( - // Function.prototype.toString - // ).isPrototypeOf(ctx.toString); // eslint-disable-line no-prototype-builtins - // if (!hasSameProto) { - // // Pass the call on to the local Function.prototype.toString instead - // return ctx.toString(); - // } + // // Check if the toString protype of the context is the same as the global prototype, + // // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case + // const hasSameProto = Object.getPrototypeOf( + // Function.prototype.toString + // ).isPrototypeOf(ctx.toString); // eslint-disable-line no-prototype-builtins + // if (!hasSameProto) { + // // Pass the call on to the local Function.prototype.toString instead + // return ctx.toString(); + // } - // return target.call(ctx); - // } - // }; + // return target.call(ctx); + // } + // }; - // const toStringProxy = new Proxy( - // Function.prototype.toString, - // utils.stripProxyFromErrors(handler) - // ); - // utils.replaceProperty(Function.prototype, 'toString', { - // value: toStringProxy - // }); + // const toStringProxy = new Proxy( + // Function.prototype.toString, + // utils.stripProxyFromErrors(handler) + // ); + // utils.replaceProperty(Function.prototype, 'toString', { + // value: toStringProxy + // }); + }; + + /** + * Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw. + * + * The presence of a JS Proxy can be revealed as it shows up in error stack traces. + * + * @param {object} handler - The JS Proxy handler to wrap + */ + utils.stripProxyFromErrors = (handler = {}) => { + const newHandler = { + setPrototypeOf: function (target, proto) { + if (proto === null) + throw new TypeError("Cannot convert object to primitive value"); + if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) { + throw new TypeError("Cyclic __proto__ value"); + } + return Reflect.setPrototypeOf(target, proto); + }, }; + // We wrap each trap in the handler in a try/catch and modify the error stack if they throw + const traps = Object.getOwnPropertyNames(handler); + traps.forEach((trap) => { + newHandler[trap] = function () { + try { + // Forward the call to the defined proxy handler + return handler[trap].call(this, ...(arguments || [])); + } catch (err) { + // Stack traces differ per browser, we only support chromium based ones currently + if (!err || !err.stack || !err.stack.includes(`at `)) { + throw err; + } - /** - * Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw. - * - * The presence of a JS Proxy can be revealed as it shows up in error stack traces. - * - * @param {object} handler - The JS Proxy handler to wrap - */ - utils.stripProxyFromErrors = (handler = {}) => { - const newHandler = { - setPrototypeOf: function (target, proto) { - if (proto === null) - throw new TypeError('Cannot convert object to primitive value'); - if (Object.getPrototypeOf(target) === Object.getPrototypeOf(proto)) { - throw new TypeError('Cyclic __proto__ value'); - } - return Reflect.setPrototypeOf(target, proto); + // When something throws within one of our traps the Proxy will show up in error stacks + // An earlier implementation of this code would simply strip lines with a blacklist, + // but it makes sense to be more surgical here and only remove lines related to our Proxy. + // We try to use a known "anchor" line for that and strip it with everything above it. + // If the anchor line cannot be found for some reason we fall back to our blacklist approach. + + const stripWithBlacklist = (stack, stripFirstLine = true) => { + const blacklist = [ + `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply + `at Object.${trap} `, // e.g. Object.get or Object.apply + `at Object.newHandler. [as ${trap}] `, // caused by this very wrapper :-) + ]; + return ( + err.stack + .split("\n") + // Always remove the first (file) line in the stack (guaranteed to be our proxy) + .filter((line, index) => !(index === 1 && stripFirstLine)) + // Check if the line starts with one of our blacklisted strings + .filter( + (line) => !blacklist.some((bl) => line.trim().startsWith(bl)), + ) + .join("\n") + ); + }; + + const stripWithAnchor = (stack, anchor) => { + const stackArr = stack.split("\n"); + anchor = anchor || `at Object.newHandler. [as ${trap}] `; // Known first Proxy line in chromium + const anchorIndex = stackArr.findIndex((line) => + line.trim().startsWith(anchor), + ); + if (anchorIndex === -1) { + return false; // 404, anchor not found } - }; - // We wrap each trap in the handler in a try/catch and modify the error stack if they throw - const traps = Object.getOwnPropertyNames(handler); - traps.forEach(trap => { - newHandler[trap] = function () { - try { - // Forward the call to the defined proxy handler - return handler[trap].call(this, ...(arguments || [])); - } catch (err) { - // Stack traces differ per browser, we only support chromium based ones currently - if (!err || !err.stack || !err.stack.includes(`at `)) { - throw err; - } + // Strip everything from the top until we reach the anchor line + // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) + stackArr.splice(1, anchorIndex); + return stackArr.join("\n"); + }; - // When something throws within one of our traps the Proxy will show up in error stacks - // An earlier implementation of this code would simply strip lines with a blacklist, - // but it makes sense to be more surgical here and only remove lines related to our Proxy. - // We try to use a known "anchor" line for that and strip it with everything above it. - // If the anchor line cannot be found for some reason we fall back to our blacklist approach. + // Special cases due to our nested toString proxies + err.stack = err.stack.replace( + "at Object.toString (", + "at Function.toString (", + ); + if ((err.stack || "").includes("at Function.toString (")) { + err.stack = stripWithBlacklist(err.stack, false); + throw err; + } - const stripWithBlacklist = (stack, stripFirstLine = true) => { - const blacklist = [ - `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply - `at Object.${trap} `, // e.g. Object.get or Object.apply - `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-) - ]; - return ( - err.stack - .split('\n') - // Always remove the first (file) line in the stack (guaranteed to be our proxy) - .filter((line, index) => !(index === 1 && stripFirstLine)) - // Check if the line starts with one of our blacklisted strings - .filter(line => !blacklist.some(bl => line.trim().startsWith(bl))) - .join('\n') - ); - }; + // Try using the anchor method, fallback to blacklist if necessary + err.stack = + stripWithAnchor(err.stack) || stripWithBlacklist(err.stack); - const stripWithAnchor = (stack, anchor) => { - const stackArr = stack.split('\n'); - anchor = anchor || `at Object.newHandler. [as ${trap}] `; // Known first Proxy line in chromium - const anchorIndex = stackArr.findIndex(line => - line.trim().startsWith(anchor) - ); - if (anchorIndex === -1) { - return false; // 404, anchor not found - } - // Strip everything from the top until we reach the anchor line - // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) - stackArr.splice(1, anchorIndex); - return stackArr.join('\n'); - }; - - // Special cases due to our nested toString proxies - err.stack = err.stack.replace( - 'at Object.toString (', - 'at Function.toString (' - ); - if ((err.stack || '').includes('at Function.toString (')) { - err.stack = stripWithBlacklist(err.stack, false); - throw err; - } - - // Try using the anchor method, fallback to blacklist if necessary - err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack); - - throw err; // Re-throw our now sanitized error - } - }; - }); - return newHandler; - }; - - /** - * Strip error lines from stack traces until (and including) a known line the stack. - * - * @param {object} err - The error to sanitize - * @param {string} anchor - The string the anchor line starts with - */ - utils.stripErrorWithAnchor = (err, anchor) => { - const stackArr = err.stack.split('\n'); - const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor)); - if (anchorIndex === -1) { - return err; // 404, anchor not found + throw err; // Re-throw our now sanitized error } - // Strip everything from the top until we reach the anchor line (remove anchor line as well) - // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) - stackArr.splice(1, anchorIndex); - err.stack = stackArr.join('\n'); - return err; - }; - - /** - * Replace the property of an object in a stealthy way. - * - * Note: You also want to work on the prototype of an object most often, - * as you'd otherwise leave traces (e.g. showing up in Object.getOwnPropertyNames(obj)). - * - * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty - * - * @example - * replaceProperty(WebGLRenderingContext.prototype, 'getParameter', { value: "alice" }) - * // or - * replaceProperty(Object.getPrototypeOf(navigator), 'languages', { get: () => ['en-US', 'en'] }) - * - * @param {object} obj - The object which has the property to replace - * @param {string} propName - The property name to replace - * @param {object} descriptorOverrides - e.g. { value: "alice" } - */ - utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => { - return Object.defineProperty(obj, propName, { - // Copy over the existing descriptors (writable, enumerable, configurable, etc) - ...(Object.getOwnPropertyDescriptor(obj, propName) || {}), - // Add our overrides (e.g. value, get()) - ...descriptorOverrides - }); - }; - - /** - * Preload a cache of function copies and data. - * - * For a determined enough observer it would be possible to overwrite and sniff usage of functions - * we use in our internal Proxies, to combat that we use a cached copy of those functions. - * - * Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before, - * by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups). - * - * This is evaluated once per execution context (e.g. window) - */ - utils.preloadCache = () => { - if (utils.cache) { - return; - } - utils.cache = { - // Used in our proxies - Reflect: { - get: Reflect.get.bind(Reflect), - apply: Reflect.apply.bind(Reflect) - }, - // Used in `makeNativeString` - nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }` - }; - }; - - /** - * Utility function to generate a cross-browser `toString` result representing native code. - * - * There's small differences: Chromium uses a single line, whereas FF & Webkit uses multiline strings. - * To future-proof this we use an existing native toString result as the basis. - * - * The only advantage we have over the other team is that our JS runs first, hence we cache the result - * of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it. - * - * @example - * makeNativeString('foobar') // => `function foobar() { [native code] }` - * - * @param {string} [name] - Optional function name - */ - utils.makeNativeString = (name = '') => { - return utils.cache.nativeToStringStr.replace('toString', name || ''); - }; - - /** - * Helper function to modify the `toString()` result of the provided object. - * - * Note: Use `utils.redirectToString` instead when possible. - * - * There's a quirk in JS Proxies that will cause the `toString()` result to differ from the vanilla Object. - * If no string is provided we will generate a `[native code]` thing based on the name of the property object. - * - * @example - * patchToString(WebGLRenderingContext.prototype.getParameter, 'function getParameter() { [native code] }') - * - * @param {object} obj - The object for which to modify the `toString()` representation - * @param {string} str - Optional string used as a return value - */ - utils.patchToString = (obj, str = '') => { - // toStringRedirects.set(obj, str); - Object.defineProperty(obj, 'toString', { - value: ()=> str, - enumerable: false, - writable: true, - configurable: true, - }); - }; - - /** - * Make all nested functions of an object native. - * - * @param {object} obj - */ - utils.patchToStringNested = (obj = {}) => { - return utils.execRecursively(obj, ['function'], utils.patchToString); - }; - - /** - * Redirect toString requests from one object to another. - * - * @param {object} proxyObj - The object that toString will be called on - * @param {object} originalObj - The object which toString result we wan to return - */ - utils.redirectToString = (proxyObj, originalObj) => { - // toStringRedirects.set(proxyObj, originalObj); - Object.defineProperty(proxyObj, 'toString', { - value: ()=> originalObj.toString(), - enumerable: false, - writable: true, - configurable: true, - }); - }; - - - /** - * All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps. - * - * Will stealthify these aspects (strip error stack traces, redirect toString, etc). - * Note: This is meant to modify native Browser APIs and works best with prototype objects. - * - * @example - * replaceWithProxy(WebGLRenderingContext.prototype, 'getParameter', proxyHandler) - * - * @param {object} obj - The object which has the property to replace - * @param {string} propName - The name of the property to replace - * @param {object} handler - The JS Proxy handler to use - */ - utils.replaceWithProxy = (obj, propName, handler) => { - const originalObj = obj[propName]; - const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler)); - - utils.replaceProperty(obj, propName, { value: proxyObj }); - utils.redirectToString(proxyObj, originalObj); - - return true; - }; - /** - * All-in-one method to replace a getter with a JS Proxy using the provided Proxy handler with traps. - * - * @example - * replaceGetterWithProxy(Object.getPrototypeOf(navigator), 'vendor', proxyHandler) - * - * @param {object} obj - The object which has the property to replace - * @param {string} propName - The name of the property to replace - * @param {object} handler - The JS Proxy handler to use - */ - utils.replaceGetterWithProxy = (obj, propName, handler) => { - const fn = Object.getOwnPropertyDescriptor(obj, propName).get; - const fnStr = fn.toString(); // special getter function string - const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler)); - - utils.replaceProperty(obj, propName, { get: proxyObj }); - utils.patchToString(proxyObj, fnStr); - - return true; - }; - - /** - * All-in-one method to replace a getter and/or setter. Functions get and set - * of handler have one more argument that contains the native function. - * - * @example - * replaceGetterSetter(HTMLIFrameElement.prototype, 'contentWindow', handler) - * - * @param {object} obj - The object which has the property to replace - * @param {string} propName - The name of the property to replace - * @param {object} handlerGetterSetter - The handler with get and/or set - * functions - * @see https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty#description - */ - utils.replaceGetterSetter = (obj, propName, handlerGetterSetter) => { - const ownPropertyDescriptor = Object.getOwnPropertyDescriptor(obj, propName); - const handler = { ...ownPropertyDescriptor }; - - if (handlerGetterSetter.get !== undefined) { - const nativeFn = ownPropertyDescriptor.get; - handler.get = function () { - return handlerGetterSetter.get.call(this, nativeFn.bind(this)); - }; - utils.redirectToString(handler.get, nativeFn); - } - - if (handlerGetterSetter.set !== undefined) { - const nativeFn = ownPropertyDescriptor.set; - handler.set = function (newValue) { - handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this)); - }; - utils.redirectToString(handler.set, nativeFn); - } - - Object.defineProperty(obj, propName, handler); - }; - - /** - * All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps. - * - * Will stealthify these aspects (strip error stack traces, redirect toString, etc). - * - * @example - * mockWithProxy(chrome.runtime, 'sendMessage', function sendMessage() {}, proxyHandler) - * - * @param {object} obj - The object which has the property to replace - * @param {string} propName - The name of the property to replace or create - * @param {object} pseudoTarget - The JS Proxy target to use as a basis - * @param {object} handler - The JS Proxy handler to use - */ - utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => { - const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)); - - utils.replaceProperty(obj, propName, { value: proxyObj }); - utils.patchToString(proxyObj); - - return true; - }; - - /** - * All-in-one method to create a new JS Proxy with stealth tweaks. - * - * This is meant to be used whenever we need a JS Proxy but don't want to replace or mock an existing known property. - * - * Will stealthify certain aspects of the Proxy (strip error stack traces, redirect toString, etc). - * - * @example - * createProxy(navigator.mimeTypes.__proto__.namedItem, proxyHandler) // => Proxy - * - * @param {object} pseudoTarget - The JS Proxy target to use as a basis - * @param {object} handler - The JS Proxy handler to use - */ - utils.createProxy = (pseudoTarget, handler) => { - const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)); - utils.patchToString(proxyObj); - - return proxyObj; - }; - - /** - * Helper function to split a full path to an Object into the first part and property. - * - * @example - * splitObjPath(`HTMLMediaElement.prototype.canPlayType`) - * // => {objName: "HTMLMediaElement.prototype", propName: "canPlayType"} - * - * @param {string} objPath - The full path to an object as dot notation string - */ - utils.splitObjPath = objPath => ({ - // Remove last dot entry (property) ==> `HTMLMediaElement.prototype` - objName: objPath.split('.').slice(0, -1).join('.'), - // Extract last dot entry ==> `canPlayType` - propName: objPath.split('.').slice(-1)[0] + }; }); + return newHandler; + }; - /** - * Convenience method to replace a property with a JS Proxy using the provided objPath. - * - * Supports a full path (dot notation) to the object as string here, in case that makes it easier. - * - * @example - * replaceObjPathWithProxy('WebGLRenderingContext.prototype.getParameter', proxyHandler) - * - * @param {string} objPath - The full path to an object (dot notation string) to replace - * @param {object} handler - The JS Proxy handler to use - */ - utils.replaceObjPathWithProxy = (objPath, handler) => { - const { objName, propName } = utils.splitObjPath(objPath); - const obj = eval(objName); - return utils.replaceWithProxy(obj, propName, handler); + /** + * Strip error lines from stack traces until (and including) a known line the stack. + * + * @param {object} err - The error to sanitize + * @param {string} anchor - The string the anchor line starts with + */ + utils.stripErrorWithAnchor = (err, anchor) => { + const stackArr = err.stack.split("\n"); + const anchorIndex = stackArr.findIndex((line) => + line.trim().startsWith(anchor), + ); + if (anchorIndex === -1) { + return err; // 404, anchor not found + } + // Strip everything from the top until we reach the anchor line (remove anchor line as well) + // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) + stackArr.splice(1, anchorIndex); + err.stack = stackArr.join("\n"); + return err; + }; + + /** + * Replace the property of an object in a stealthy way. + * + * Note: You also want to work on the prototype of an object most often, + * as you'd otherwise leave traces (e.g. showing up in Object.getOwnPropertyNames(obj)). + * + * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty + * + * @example + * replaceProperty(WebGLRenderingContext.prototype, 'getParameter', { value: "alice" }) + * // or + * replaceProperty(Object.getPrototypeOf(navigator), 'languages', { get: () => ['en-US', 'en'] }) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The property name to replace + * @param {object} descriptorOverrides - e.g. { value: "alice" } + */ + utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => { + return Object.defineProperty(obj, propName, { + // Copy over the existing descriptors (writable, enumerable, configurable, etc) + ...(Object.getOwnPropertyDescriptor(obj, propName) || {}), + // Add our overrides (e.g. value, get()) + ...descriptorOverrides, + }); + }; + + /** + * Preload a cache of function copies and data. + * + * For a determined enough observer it would be possible to overwrite and sniff usage of functions + * we use in our internal Proxies, to combat that we use a cached copy of those functions. + * + * Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before, + * by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups). + * + * This is evaluated once per execution context (e.g. window) + */ + utils.preloadCache = () => { + if (utils.cache) { + return; + } + utils.cache = { + // Used in our proxies + Reflect: { + get: Reflect.get.bind(Reflect), + apply: Reflect.apply.bind(Reflect), + }, + // Used in `makeNativeString` + nativeToStringStr: Function.toString + "", // => `function toString() { [native code] }` }; + }; - /** - * Traverse nested properties of an object recursively and apply the given function on a whitelist of value types. - * - * @param {object} obj - * @param {array} typeFilter - e.g. `['function']` - * @param {Function} fn - e.g. `utils.patchToString` - */ - utils.execRecursively = (obj = {}, typeFilter = [], fn) => { - function recurse(obj) { - for (const key in obj) { - if (obj[key] === undefined) { - continue; - } - if (obj[key] && typeof obj[key] === 'object') { - recurse(obj[key]); - } else { - if (obj[key] && typeFilter.includes(typeof obj[key])) { - fn.call(this, obj[key]); - } - } - } + /** + * Utility function to generate a cross-browser `toString` result representing native code. + * + * There's small differences: Chromium uses a single line, whereas FF & Webkit uses multiline strings. + * To future-proof this we use an existing native toString result as the basis. + * + * The only advantage we have over the other team is that our JS runs first, hence we cache the result + * of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it. + * + * @example + * makeNativeString('foobar') // => `function foobar() { [native code] }` + * + * @param {string} [name] - Optional function name + */ + utils.makeNativeString = (name = "") => { + return utils.cache.nativeToStringStr.replace("toString", name || ""); + }; + + /** + * Helper function to modify the `toString()` result of the provided object. + * + * Note: Use `utils.redirectToString` instead when possible. + * + * There's a quirk in JS Proxies that will cause the `toString()` result to differ from the vanilla Object. + * If no string is provided we will generate a `[native code]` thing based on the name of the property object. + * + * @example + * patchToString(WebGLRenderingContext.prototype.getParameter, 'function getParameter() { [native code] }') + * + * @param {object} obj - The object for which to modify the `toString()` representation + * @param {string} str - Optional string used as a return value + */ + utils.patchToString = (obj, str = "") => { + // toStringRedirects.set(obj, str); + Object.defineProperty(obj, "toString", { + value: () => str, + enumerable: false, + writable: true, + configurable: true, + }); + }; + + /** + * Make all nested functions of an object native. + * + * @param {object} obj + */ + utils.patchToStringNested = (obj = {}) => { + return utils.execRecursively(obj, ["function"], utils.patchToString); + }; + + /** + * Redirect toString requests from one object to another. + * + * @param {object} proxyObj - The object that toString will be called on + * @param {object} originalObj - The object which toString result we wan to return + */ + utils.redirectToString = (proxyObj, originalObj) => { + // toStringRedirects.set(proxyObj, originalObj); + Object.defineProperty(proxyObj, "toString", { + value: () => originalObj.toString(), + enumerable: false, + writable: true, + configurable: true, + }); + }; + + /** + * All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps. + * + * Will stealthify these aspects (strip error stack traces, redirect toString, etc). + * Note: This is meant to modify native Browser APIs and works best with prototype objects. + * + * @example + * replaceWithProxy(WebGLRenderingContext.prototype, 'getParameter', proxyHandler) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The name of the property to replace + * @param {object} handler - The JS Proxy handler to use + */ + utils.replaceWithProxy = (obj, propName, handler) => { + const originalObj = obj[propName]; + const proxyObj = new Proxy( + obj[propName], + utils.stripProxyFromErrors(handler), + ); + + utils.replaceProperty(obj, propName, { value: proxyObj }); + utils.redirectToString(proxyObj, originalObj); + + return true; + }; + /** + * All-in-one method to replace a getter with a JS Proxy using the provided Proxy handler with traps. + * + * @example + * replaceGetterWithProxy(Object.getPrototypeOf(navigator), 'vendor', proxyHandler) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The name of the property to replace + * @param {object} handler - The JS Proxy handler to use + */ + utils.replaceGetterWithProxy = (obj, propName, handler) => { + const fn = Object.getOwnPropertyDescriptor(obj, propName).get; + const fnStr = fn.toString(); // special getter function string + const proxyObj = new Proxy(fn, utils.stripProxyFromErrors(handler)); + + utils.replaceProperty(obj, propName, { get: proxyObj }); + utils.patchToString(proxyObj, fnStr); + + return true; + }; + + /** + * All-in-one method to replace a getter and/or setter. Functions get and set + * of handler have one more argument that contains the native function. + * + * @example + * replaceGetterSetter(HTMLIFrameElement.prototype, 'contentWindow', handler) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The name of the property to replace + * @param {object} handlerGetterSetter - The handler with get and/or set + * functions + * @see https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty#description + */ + utils.replaceGetterSetter = (obj, propName, handlerGetterSetter) => { + const ownPropertyDescriptor = Object.getOwnPropertyDescriptor( + obj, + propName, + ); + const handler = { ...ownPropertyDescriptor }; + + if (handlerGetterSetter.get !== undefined) { + const nativeFn = ownPropertyDescriptor.get; + handler.get = function () { + return handlerGetterSetter.get.call(this, nativeFn.bind(this)); + }; + utils.redirectToString(handler.get, nativeFn); + } + + if (handlerGetterSetter.set !== undefined) { + const nativeFn = ownPropertyDescriptor.set; + handler.set = function (newValue) { + handlerGetterSetter.set.call(this, newValue, nativeFn.bind(this)); + }; + utils.redirectToString(handler.set, nativeFn); + } + + Object.defineProperty(obj, propName, handler); + }; + + /** + * All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps. + * + * Will stealthify these aspects (strip error stack traces, redirect toString, etc). + * + * @example + * mockWithProxy(chrome.runtime, 'sendMessage', function sendMessage() {}, proxyHandler) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The name of the property to replace or create + * @param {object} pseudoTarget - The JS Proxy target to use as a basis + * @param {object} handler - The JS Proxy handler to use + */ + utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => { + const proxyObj = new Proxy( + pseudoTarget, + utils.stripProxyFromErrors(handler), + ); + + utils.replaceProperty(obj, propName, { value: proxyObj }); + utils.patchToString(proxyObj); + + return true; + }; + + /** + * All-in-one method to create a new JS Proxy with stealth tweaks. + * + * This is meant to be used whenever we need a JS Proxy but don't want to replace or mock an existing known property. + * + * Will stealthify certain aspects of the Proxy (strip error stack traces, redirect toString, etc). + * + * @example + * createProxy(navigator.mimeTypes.__proto__.namedItem, proxyHandler) // => Proxy + * + * @param {object} pseudoTarget - The JS Proxy target to use as a basis + * @param {object} handler - The JS Proxy handler to use + */ + utils.createProxy = (pseudoTarget, handler) => { + const proxyObj = new Proxy( + pseudoTarget, + utils.stripProxyFromErrors(handler), + ); + utils.patchToString(proxyObj); + + return proxyObj; + }; + + /** + * Helper function to split a full path to an Object into the first part and property. + * + * @example + * splitObjPath(`HTMLMediaElement.prototype.canPlayType`) + * // => {objName: "HTMLMediaElement.prototype", propName: "canPlayType"} + * + * @param {string} objPath - The full path to an object as dot notation string + */ + utils.splitObjPath = (objPath) => ({ + // Remove last dot entry (property) ==> `HTMLMediaElement.prototype` + objName: objPath.split(".").slice(0, -1).join("."), + // Extract last dot entry ==> `canPlayType` + propName: objPath.split(".").slice(-1)[0], + }); + + /** + * Convenience method to replace a property with a JS Proxy using the provided objPath. + * + * Supports a full path (dot notation) to the object as string here, in case that makes it easier. + * + * @example + * replaceObjPathWithProxy('WebGLRenderingContext.prototype.getParameter', proxyHandler) + * + * @param {string} objPath - The full path to an object (dot notation string) to replace + * @param {object} handler - The JS Proxy handler to use + */ + utils.replaceObjPathWithProxy = (objPath, handler) => { + const { objName, propName } = utils.splitObjPath(objPath); + const obj = eval(objName); + return utils.replaceWithProxy(obj, propName, handler); + }; + + /** + * Traverse nested properties of an object recursively and apply the given function on a whitelist of value types. + * + * @param {object} obj + * @param {array} typeFilter - e.g. `['function']` + * @param {Function} fn - e.g. `utils.patchToString` + */ + utils.execRecursively = (obj = {}, typeFilter = [], fn) => { + function recurse(obj) { + for (const key in obj) { + if (obj[key] === undefined) { + continue; } - recurse(obj); + if (obj[key] && typeof obj[key] === "object") { + recurse(obj[key]); + } else { + if (obj[key] && typeFilter.includes(typeof obj[key])) { + fn.call(this, obj[key]); + } + } + } + } + recurse(obj); + return obj; + }; + + /** + * Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one. + * That means we cannot just use reference variables and functions from outside code, we need to pass everything as a parameter. + * + * Unfortunately the data we can pass is only allowed to be of primitive types, regular functions don't survive the built-in serialization process. + * This utility function will take an object with functions and stringify them, so we can pass them down unharmed as strings. + * + * We use this to pass down our utility functions as well as any other functions (to be able to split up code better). + * + * @see utils.materializeFns + * + * @param {object} fnObj - An object containing functions as properties + */ + utils.stringifyFns = (fnObj = { hello: () => "world" }) => { + // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine + // https://github.com/feross/fromentries + function fromEntries(iterable) { + return [...iterable].reduce((obj, [key, val]) => { + obj[key] = val; return obj; - }; - - /** - * Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one. - * That means we cannot just use reference variables and functions from outside code, we need to pass everything as a parameter. - * - * Unfortunately the data we can pass is only allowed to be of primitive types, regular functions don't survive the built-in serialization process. - * This utility function will take an object with functions and stringify them, so we can pass them down unharmed as strings. - * - * We use this to pass down our utility functions as well as any other functions (to be able to split up code better). - * - * @see utils.materializeFns - * - * @param {object} fnObj - An object containing functions as properties - */ - utils.stringifyFns = (fnObj = { hello: () => 'world' }) => { - // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine - // https://github.com/feross/fromentries - function fromEntries(iterable) { - return [...iterable].reduce((obj, [key, val]) => { - obj[key] = val; - return obj; - }, {}); - } - return (Object.fromEntries || fromEntries)( - Object.entries(fnObj) - .filter(([, value]) => typeof value === 'function') - .map(([key, value]) => [key, value.toString()]) - ); - }; - - /** - * Utility function to reverse the process of `utils.stringifyFns`. - * Will materialize an object with stringified functions (supports classic and fat arrow functions). - * - * @param {object} fnStrObj - An object containing stringified functions as properties - */ - utils.materializeFns = (fnStrObj = { hello: "() => 'world'" }) => { - return Object.fromEntries( - Object.entries(fnStrObj).map(([key, value]) => { - if (value.startsWith('function')) { - // some trickery is needed to make oldschool functions work :-) - return [key, eval(`() => ${value}`)()]; - } else { - // arrow functions just work - return [key, eval(value)]; - } - }) - ); - }; - - // Proxy handler templates for re-usability - utils.makeHandler = () => ({ - // Used by simple `navigator` getter evasions - getterValue: value => ({ - apply(_target, _ctx, _args) { - // Let's fetch the value first, to trigger and escalate potential errors - // Illegal invocations like `navigator.__proto__.vendor` will throw here - utils.cache.Reflect.apply(...arguments); - return value; - } - }) - }); - - /** - * Compare two arrays. - * - * @param {array} array1 - First array - * @param {array} array2 - Second array - */ - utils.arrayEquals = (array1, array2) => { - if (array1.length !== array2.length) { - return false; - } - for (let i = 0; i < array1.length; ++i) { - if (array1[i] !== array2[i]) { - return false; - } - } - return true; - }; - - /** - * Cache the method return according to its arguments. - * - * @param {Function} fn - A function that will be cached - */ - utils.memoize = fn => { - const cache = []; - return function (...args) { - if (!cache.some(c => utils.arrayEquals(c.key, args))) { - cache.push({ key: args, value: fn.apply(this, args) }); - } - return cache.find(c => utils.arrayEquals(c.key, args)).value; - }; - }; - - utils.init(); - - const getParameterProxyHandler = { - apply: function (target, ctx, args) { - const param = (args || [])[0] - const result = utils.cache.Reflect.apply(target, ctx, args) - // UNMASKED_VENDOR_WEBGL - if (param === 37445) { - return 'Intel Inc.' // default in headless: Google Inc. - } - // UNMASKED_RENDERER_WEBGL - if (param === 37446) { - return 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader - } - return result - } + }, {}); } + return (Object.fromEntries || fromEntries)( + Object.entries(fnObj) + .filter(([, value]) => typeof value === "function") + .map(([key, value]) => [key, value.toString()]), + ); + }; - // There's more than one WebGL rendering context - // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility - // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter) - const addProxy = (obj, propName) => { - utils.replaceWithProxy(obj, propName, getParameterProxyHandler) + /** + * Utility function to reverse the process of `utils.stringifyFns`. + * Will materialize an object with stringified functions (supports classic and fat arrow functions). + * + * @param {object} fnStrObj - An object containing stringified functions as properties + */ + utils.materializeFns = (fnStrObj = { hello: "() => 'world'" }) => { + return Object.fromEntries( + Object.entries(fnStrObj).map(([key, value]) => { + if (value.startsWith("function")) { + // some trickery is needed to make oldschool functions work :-) + return [key, eval(`() => ${value}`)()]; + } else { + // arrow functions just work + return [key, eval(value)]; + } + }), + ); + }; + + // Proxy handler templates for re-usability + utils.makeHandler = () => ({ + // Used by simple `navigator` getter evasions + getterValue: (value) => ({ + apply(_target, _ctx, _args) { + // Let's fetch the value first, to trigger and escalate potential errors + // Illegal invocations like `navigator.__proto__.vendor` will throw here + utils.cache.Reflect.apply(...arguments); + return value; + }, + }), + }); + + /** + * Compare two arrays. + * + * @param {array} array1 - First array + * @param {array} array2 - Second array + */ + utils.arrayEquals = (array1, array2) => { + if (array1.length !== array2.length) { + return false; } - // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing: - addProxy(WebGLRenderingContext.prototype, 'getParameter') - addProxy(WebGL2RenderingContext.prototype, 'getParameter') + for (let i = 0; i < array1.length; ++i) { + if (array1[i] !== array2[i]) { + return false; + } + } + return true; + }; + /** + * Cache the method return according to its arguments. + * + * @param {Function} fn - A function that will be cached + */ + utils.memoize = (fn) => { + const cache = []; + return function (...args) { + if (!cache.some((c) => utils.arrayEquals(c.key, args))) { + cache.push({ key: args, value: fn.apply(this, args) }); + } + return cache.find((c) => utils.arrayEquals(c.key, args)).value; + }; + }; + + utils.init(); + + const getParameterProxyHandler = { + apply: function (target, ctx, args) { + const param = (args || [])[0]; + const result = utils.cache.Reflect.apply(target, ctx, args); + // UNMASKED_VENDOR_WEBGL + if (param === 37445) { + return "Intel Inc."; // default in headless: Google Inc. + } + // UNMASKED_RENDERER_WEBGL + if (param === 37446) { + return "Intel Iris OpenGL Engine"; // default in headless: Google SwiftShader + } + return result; + }, + }; + + // There's more than one WebGL rendering context + // https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility + // To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter) + const addProxy = (obj, propName) => { + utils.replaceWithProxy(obj, propName, getParameterProxyHandler); + }; + // For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing: + addProxy(WebGLRenderingContext.prototype, "getParameter"); + addProxy(WebGL2RenderingContext.prototype, "getParameter"); } diff --git a/src/services/misc.ts b/src/services/misc.ts index 6fa2012..0a56735 100644 --- a/src/services/misc.ts +++ b/src/services/misc.ts @@ -1,129 +1,140 @@ -import { singleton } from 'tsyringe'; -import { AsyncService } from 'civkit/async-service'; -import { ParamValidationError } from 'civkit/civ-rpc'; -import { SecurityCompromiseError } from '../shared/lib/errors'; -import { isIP } from 'node:net'; -import { isIPInNonPublicRange } from '../utils/ip'; -import { GlobalLogger } from './logger'; -import { lookup } from 'node:dns/promises'; -import { Threaded } from './threaded'; +import { singleton } from "tsyringe"; +import { AsyncService } from "civkit/async-service"; +import { ParamValidationError } from "civkit/civ-rpc"; +import { SecurityCompromiseError } from "../shared/lib/errors"; +import { isIP } from "node:net"; +import { isIPInNonPublicRange } from "../utils/ip"; +import { GlobalLogger } from "./logger"; +import { lookup } from "node:dns/promises"; +import { Threaded } from "./threaded"; -const normalizeUrl = require('@esm2cjs/normalize-url').default; +const normalizeUrl = require("@esm2cjs/normalize-url").default; -const STANDALONE_ALLOW_INTERNAL_DNS_REWRITE = process.env.STANDALONE_ALLOW_INTERNAL_DNS_REWRITE === 'true'; +const STANDALONE_ALLOW_INTERNAL_DNS_REWRITE = + process.env.STANDALONE_ALLOW_INTERNAL_DNS_REWRITE === "true"; function isStandaloneDnsRewriteAddress(address: string) { - const normalized = address.toLowerCase(); + const normalized = address.toLowerCase(); - return normalized.startsWith('198.18.') - || normalized.startsWith('198.19.') - || normalized.startsWith('fd00::'); + return ( + normalized.startsWith("198.18.") || + normalized.startsWith("198.19.") || + normalized.startsWith("fd00::") + ); } @singleton() export class MiscService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + constructor(protected globalLogger: GlobalLogger) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, + override async init() { + await this.dependencyReady(); + + this.emit("ready"); + } + + @Threaded() + async assertNormalizedUrl(input: string) { + let result: URL; + try { + result = new URL( + normalizeUrl(input, { + stripWWW: false, + removeTrailingSlash: false, + removeSingleSlash: false, + sortQueryParameters: false, + }), + ); + } catch (err) { + throw new ParamValidationError({ + message: `${err}`, + path: "url", + }); + } + + if (!["http:", "https:", "blob:"].includes(result.protocol)) { + throw new ParamValidationError({ + message: `Invalid protocol ${result.protocol}`, + path: "url", + }); + } + + const normalizedHostname = result.hostname.startsWith("[") + ? result.hostname.slice(1, -1) + : result.hostname; + let ips: string[] = []; + const isIp = isIP(normalizedHostname); + if (isIp) { + ips.push(normalizedHostname); + } + if ( + result.hostname === "localhost" || + (isIp && isIPInNonPublicRange(normalizedHostname)) ) { - super(...arguments); + this.logger.warn( + `Suspicious action: Request to localhost or non-public IP: ${normalizedHostname}`, + { href: result.href }, + ); + throw new SecurityCompromiseError({ + message: `Suspicious action: Request to localhost or non-public IP: ${normalizedHostname}`, + path: "url", + }); } - - override async init() { - await this.dependencyReady(); - - this.emit('ready'); - } - - @Threaded() - async assertNormalizedUrl(input: string) { - let result: URL; - try { - result = new URL( - normalizeUrl( - input, - { - stripWWW: false, - removeTrailingSlash: false, - removeSingleSlash: false, - sortQueryParameters: false, - } - ) + if (!isIp && result.protocol !== "blob:") { + const resolved = await lookup(result.hostname, { all: true }).catch( + (err) => { + if (err.code === "ENOTFOUND") { + return Promise.reject( + new ParamValidationError({ + message: `Domain '${result.hostname}' could not be resolved`, + path: "url", + }), ); - } catch (err) { - throw new ParamValidationError({ - message: `${err}`, - path: 'url' - }); + } + + return; + }, + ); + if (resolved) { + const resolvedAddrs = resolved.map((x) => x.address); + const allowStandaloneDnsRewrite = + STANDALONE_ALLOW_INTERNAL_DNS_REWRITE && + resolvedAddrs.length && + resolvedAddrs.every((x) => isStandaloneDnsRewriteAddress(x)); + + if (allowStandaloneDnsRewrite) { + this.logger.warn( + `Allowing Docker DNS rewrite addresses for standalone container`, + { + href: result.href, + ips: resolvedAddrs, + }, + ); } - if (!['http:', 'https:', 'blob:'].includes(result.protocol)) { - throw new ParamValidationError({ - message: `Invalid protocol ${result.protocol}`, - path: 'url' - }); - } - - const normalizedHostname = result.hostname.startsWith('[') ? result.hostname.slice(1, -1) : result.hostname; - let ips: string[] = []; - const isIp = isIP(normalizedHostname); - if (isIp) { - ips.push(normalizedHostname); - } - if ( - (result.hostname === 'localhost') || - (isIp && isIPInNonPublicRange(normalizedHostname)) - ) { - this.logger.warn(`Suspicious action: Request to localhost or non-public IP: ${normalizedHostname}`, { href: result.href }); + for (const x of resolved) { + if (isIPInNonPublicRange(x.address) && !allowStandaloneDnsRewrite) { + this.logger.warn( + `Suspicious action: Domain resolved to non-public IP: ${result.hostname} => ${x.address}`, + { href: result.href, ip: x.address }, + ); throw new SecurityCompromiseError({ - message: `Suspicious action: Request to localhost or non-public IP: ${normalizedHostname}`, - path: 'url' + message: `Suspicious action: Domain resolved to non-public IP: ${x.address}`, + path: "url", }); + } + ips.push(x.address); } - if (!isIp && result.protocol !== 'blob:') { - const resolved = await lookup(result.hostname, { all: true }).catch((err) => { - if (err.code === 'ENOTFOUND') { - return Promise.reject(new ParamValidationError({ - message: `Domain '${result.hostname}' could not be resolved`, - path: 'url' - })); - } - - return; - }); - if (resolved) { - const resolvedAddrs = resolved.map((x) => x.address); - const allowStandaloneDnsRewrite = STANDALONE_ALLOW_INTERNAL_DNS_REWRITE - && resolvedAddrs.length - && resolvedAddrs.every((x) => isStandaloneDnsRewriteAddress(x)); - - if (allowStandaloneDnsRewrite) { - this.logger.warn(`Allowing Docker DNS rewrite addresses for standalone container`, { - href: result.href, - ips: resolvedAddrs, - }); - } - - for (const x of resolved) { - if (isIPInNonPublicRange(x.address) && !allowStandaloneDnsRewrite) { - this.logger.warn(`Suspicious action: Domain resolved to non-public IP: ${result.hostname} => ${x.address}`, { href: result.href, ip: x.address }); - throw new SecurityCompromiseError({ - message: `Suspicious action: Domain resolved to non-public IP: ${x.address}`, - path: 'url' - }); - } - ips.push(x.address); - } - - } - } - - return { - url: result, - ips - }; + } } + return { + url: result, + ips, + }; + } } diff --git a/src/services/pdf-extract.ts b/src/services/pdf-extract.ts index e1bbfaa..5e2d414 100644 --- a/src/services/pdf-extract.ts +++ b/src/services/pdf-extract.ts @@ -1,378 +1,436 @@ -import { singleton } from 'tsyringe'; -import _ from 'lodash'; -import { TextItem } from 'pdfjs-dist/types/src/display/api'; -import { AssertionFailureError, AsyncService, HashManager } from 'civkit'; -import { GlobalLogger } from './logger'; -import { PDFContent } from '../db/pdf'; -import dayjs from 'dayjs'; -import { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket'; -import { randomUUID } from 'crypto'; -import type { PDFDocumentLoadingTask } from 'pdfjs-dist'; -import path from 'path'; -import { AsyncLocalContext } from './async-context'; -const utc = require('dayjs/plugin/utc'); // Import the UTC plugin -dayjs.extend(utc); // Extend dayjs with the UTC plugin -const timezone = require('dayjs/plugin/timezone'); +import { singleton } from "tsyringe"; +import _ from "lodash"; +import { TextItem } from "pdfjs-dist/types/src/display/api"; +import { AssertionFailureError, AsyncService, HashManager } from "civkit"; +import { GlobalLogger } from "./logger"; +import { PDFContent } from "../db/pdf"; +import dayjs from "dayjs"; +import { FirebaseStorageBucketControl } from "../shared/services/firebase-storage-bucket"; +import { randomUUID } from "crypto"; +import type { PDFDocumentLoadingTask } from "pdfjs-dist"; +import path from "path"; +import { AsyncLocalContext } from "./async-context"; +const utc = require("dayjs/plugin/utc"); // Import the UTC plugin +dayjs.extend(utc); // Extend dayjs with the UTC plugin +const timezone = require("dayjs/plugin/timezone"); dayjs.extend(timezone); -const pPdfjs = import('pdfjs-dist/legacy/build/pdf.mjs'); -const nodeCmapUrl = path.resolve(require.resolve('pdfjs-dist'), '../../cmaps') + '/'; +const pPdfjs = import("pdfjs-dist/legacy/build/pdf.mjs"); +const nodeCmapUrl = + path.resolve(require.resolve("pdfjs-dist"), "../../cmaps") + "/"; -const md5Hasher = new HashManager('md5', 'hex'); +const md5Hasher = new HashManager("md5", "hex"); function stdDev(numbers: number[]) { - const mean = _.mean(numbers); - const squareDiffs = numbers.map((num) => Math.pow(num - mean, 2)); - const avgSquareDiff = _.mean(squareDiffs); - return Math.sqrt(avgSquareDiff); + const mean = _.mean(numbers); + const squareDiffs = numbers.map((num) => Math.pow(num - mean, 2)); + const avgSquareDiff = _.mean(squareDiffs); + return Math.sqrt(avgSquareDiff); } -function isRotatedByAtLeast35Degrees(transform?: [number, number, number, number, number, number]): boolean { - if (!transform) { - return false; - } - const [a, b, c, d, _e, _f] = transform; +function isRotatedByAtLeast35Degrees( + transform?: [number, number, number, number, number, number], +): boolean { + if (!transform) { + return false; + } + const [a, b, c, d, _e, _f] = transform; - // Calculate the rotation angles using arctan(b/a) and arctan(-c/d) - const angle1 = Math.atan2(b, a) * (180 / Math.PI); // from a, b - const angle2 = Math.atan2(-c, d) * (180 / Math.PI); // from c, d + // Calculate the rotation angles using arctan(b/a) and arctan(-c/d) + const angle1 = Math.atan2(b, a) * (180 / Math.PI); // from a, b + const angle2 = Math.atan2(-c, d) * (180 / Math.PI); // from c, d - // Either angle1 or angle2 can be used to determine the rotation, they should be equivalent - const rotationAngle1 = Math.abs(angle1); - const rotationAngle2 = Math.abs(angle2); + // Either angle1 or angle2 can be used to determine the rotation, they should be equivalent + const rotationAngle1 = Math.abs(angle1); + const rotationAngle2 = Math.abs(angle2); - // Check if the absolute rotation angle is greater than or equal to 35 degrees - return rotationAngle1 >= 35 || rotationAngle2 >= 35; + // Check if the absolute rotation angle is greater than or equal to 35 degrees + return rotationAngle1 >= 35 || rotationAngle2 >= 35; } @singleton() export class PDFExtractor extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); + pdfjs!: Awaited; - logger = this.globalLogger.child({ service: this.constructor.name }); - pdfjs!: Awaited; + cacheRetentionMs = 1000 * 3600 * 24 * 7; - cacheRetentionMs = 1000 * 3600 * 24 * 7; + constructor( + protected globalLogger: GlobalLogger, + protected firebaseObjectStorage: FirebaseStorageBucketControl, + protected asyncLocalContext: AsyncLocalContext, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected firebaseObjectStorage: FirebaseStorageBucketControl, - protected asyncLocalContext: AsyncLocalContext, - ) { - super(...arguments); + override async init() { + await this.dependencyReady(); + this.pdfjs = await pPdfjs; + + this.emit("ready"); + } + + isDataUrl(url: string) { + return url.startsWith("data:"); + } + + parseDataUrl(url: string) { + const protocol = url.slice(0, url.indexOf(":")); + const contentType = url.slice(url.indexOf(":") + 1, url.indexOf(";")); + const data = url.slice(url.indexOf(",") + 1); + if (protocol !== "data" || !data) { + throw new Error("Invalid data URL"); } - override async init() { - await this.dependencyReady(); - this.pdfjs = await pPdfjs; - - this.emit('ready'); + if (contentType !== "application/pdf") { + throw new Error("Invalid data URL type"); } - isDataUrl(url: string) { - return url.startsWith('data:'); + return { + type: contentType, + data: data, + }; + } + + async extract(url: string | URL) { + let loadingTask: PDFDocumentLoadingTask; + + if (typeof url === "string" && this.isDataUrl(url)) { + const { data } = this.parseDataUrl(url); + const binary = Uint8Array.from(Buffer.from(data, "base64")); + loadingTask = this.pdfjs.getDocument({ + data: binary, + disableFontFace: true, + verbosity: 0, + cMapUrl: nodeCmapUrl, + }); + } else { + loadingTask = this.pdfjs.getDocument({ + url, + disableFontFace: true, + verbosity: 0, + cMapUrl: nodeCmapUrl, + }); } - parseDataUrl(url: string) { - const protocol = url.slice(0, url.indexOf(':')); - const contentType = url.slice(url.indexOf(':') + 1, url.indexOf(';')); - const data = url.slice(url.indexOf(',') + 1); - if (protocol !== 'data' || !data) { - throw new Error('Invalid data URL'); + const doc = await loadingTask.promise; + const meta = await doc.getMetadata(); + + const textItems: TextItem[][] = []; + + for (const pg of _.range(0, doc.numPages)) { + const page = await doc.getPage(pg + 1); + const textContent = await page.getTextContent({ + includeMarkedContent: true, + }); + textItems.push(textContent.items as TextItem[]); + } + + const articleCharHeights: number[] = []; + for (const textItem of textItems.flat()) { + if (textItem.height) { + articleCharHeights.push( + ...Array(textItem.str.length).fill(textItem.height), + ); + } + } + const articleAvgHeight = _.mean(articleCharHeights); + const articleStdDevHeight = stdDev(articleCharHeights); + // const articleMedianHeight = articleCharHeights.sort()[Math.floor(articleCharHeights.length / 2)]; + const mdOps: Array<{ + text: string; + op?: "new" | "append"; + mode: "h1" | "h2" | "p" | "appendix" | "space"; + }> = []; + + const rawChunks: string[] = []; + + let op: "append" | "new" = "new"; + let mode: "h1" | "h2" | "p" | "space" | "appendix" = "p"; + for (const pageTextItems of textItems) { + const charHeights = []; + for (const textItem of pageTextItems as TextItem[]) { + if (textItem.height) { + charHeights.push(...Array(textItem.str.length).fill(textItem.height)); } + rawChunks.push(`${textItem.str}${textItem.hasEOL ? "\n" : ""}`); + } - if (contentType !== 'application/pdf') { - throw new Error('Invalid data URL type'); - } + const avgHeight = _.mean(charHeights); + const stdDevHeight = stdDev(charHeights); + // const medianHeight = charHeights.sort()[Math.floor(charHeights.length / 2)]; - return { - type: contentType, - data: data - }; - } - - async extract(url: string | URL) { - let loadingTask: PDFDocumentLoadingTask; - - if (typeof url === 'string' && this.isDataUrl(url)) { - const { data } = this.parseDataUrl(url); - const binary = Uint8Array.from(Buffer.from(data, 'base64')); - loadingTask = this.pdfjs.getDocument({ - data: binary, - disableFontFace: true, - verbosity: 0, - cMapUrl: nodeCmapUrl, - }); + for (const textItem of pageTextItems) { + if (textItem.height > articleAvgHeight + 3 * articleStdDevHeight) { + mode = "h1"; + } else if ( + textItem.height > + articleAvgHeight + 2 * articleStdDevHeight + ) { + mode = "h2"; + } else if ( + textItem.height && + textItem.height < avgHeight - stdDevHeight + ) { + mode = "appendix"; + } else if (textItem.height) { + mode = "p"; } else { - loadingTask = this.pdfjs.getDocument({ - url, - disableFontFace: true, - verbosity: 0, - cMapUrl: nodeCmapUrl, - }); + mode = "space"; } - - const doc = await loadingTask.promise; - const meta = await doc.getMetadata(); - - const textItems: TextItem[][] = []; - - for (const pg of _.range(0, doc.numPages)) { - const page = await doc.getPage(pg + 1); - const textContent = await page.getTextContent({ includeMarkedContent: true }); - textItems.push((textContent.items as TextItem[])); + if (isRotatedByAtLeast35Degrees(textItem.transform as any)) { + mode = "appendix"; } - const articleCharHeights: number[] = []; - for (const textItem of textItems.flat()) { - if (textItem.height) { - articleCharHeights.push(...Array(textItem.str.length).fill(textItem.height)); - } + mdOps.push({ + op, + mode, + text: textItem.str, + }); + + if (textItem.hasEOL && !textItem.str) { + op = "new"; + } else { + op = "append"; } - const articleAvgHeight = _.mean(articleCharHeights); - const articleStdDevHeight = stdDev(articleCharHeights); - // const articleMedianHeight = articleCharHeights.sort()[Math.floor(articleCharHeights.length / 2)]; - const mdOps: Array<{ - text: string; - op?: 'new' | 'append'; - mode: 'h1' | 'h2' | 'p' | 'appendix' | 'space'; - }> = []; - - const rawChunks: string[] = []; - - let op: 'append' | 'new' = 'new'; - let mode: 'h1' | 'h2' | 'p' | 'space' | 'appendix' = 'p'; - for (const pageTextItems of textItems) { - const charHeights = []; - for (const textItem of pageTextItems as TextItem[]) { - if (textItem.height) { - charHeights.push(...Array(textItem.str.length).fill(textItem.height)); - } - rawChunks.push(`${textItem.str}${textItem.hasEOL ? '\n' : ''}`); - } - - const avgHeight = _.mean(charHeights); - const stdDevHeight = stdDev(charHeights); - // const medianHeight = charHeights.sort()[Math.floor(charHeights.length / 2)]; - - for (const textItem of pageTextItems) { - if (textItem.height > articleAvgHeight + 3 * articleStdDevHeight) { - mode = 'h1'; - } else if (textItem.height > articleAvgHeight + 2 * articleStdDevHeight) { - mode = 'h2'; - } else if (textItem.height && textItem.height < avgHeight - stdDevHeight) { - mode = 'appendix'; - } else if (textItem.height) { - mode = 'p'; - } else { - mode = 'space'; - } - - if (isRotatedByAtLeast35Degrees(textItem.transform as any)) { - mode = 'appendix'; - } - - mdOps.push({ - op, - mode, - text: textItem.str - }); - - if (textItem.hasEOL && !textItem.str) { - op = 'new'; - } else { - op = 'append'; - } - } - } - - const mdChunks = []; - const appendixChunks = []; - mode = 'space'; - for (const x of mdOps) { - const previousMode: string = mode; - const changeToMdChunks = []; - - const isNewStart = x.mode !== 'space' && (x.op === 'new' || (previousMode === 'appendix' && x.mode !== previousMode)); - - if (isNewStart) { - switch (x.mode) { - case 'h1': { - changeToMdChunks.push(`\n\n# `); - mode = x.mode; - break; - } - - case 'h2': { - changeToMdChunks.push(`\n\n## `); - mode = x.mode; - break; - } - - case 'p': { - changeToMdChunks.push(`\n\n`); - mode = x.mode; - break; - } - - case 'appendix': { - mode = x.mode; - appendixChunks.push(`\n\n`); - break; - } - - default: { - break; - } - } - } else { - if (x.mode === 'appendix' && appendixChunks.length) { - const lastChunk = appendixChunks[appendixChunks.length - 1]; - if (!lastChunk.match(/(\s+|-)$/) && lastChunk.length !== 1) { - appendixChunks.push(' '); - } - } else if (mdChunks.length) { - const lastChunk = mdChunks[mdChunks.length - 1]; - if (!lastChunk.match(/(\s+|-)$/) && lastChunk.length !== 1) { - changeToMdChunks.push(' '); - } - } - } - - if (x.text) { - if (x.mode == 'appendix') { - if (appendixChunks.length || isNewStart) { - appendixChunks.push(x.text); - } else { - changeToMdChunks.push(x.text); - } - } else { - changeToMdChunks.push(x.text); - } - } - - if (isNewStart && x.mode !== 'appendix' && appendixChunks.length) { - const appendix = appendixChunks.join('').split(/\r?\n/).map((x) => x.trim()).filter(Boolean).map((x) => `> ${x}`).join('\n'); - changeToMdChunks.unshift(appendix); - changeToMdChunks.unshift(`\n\n`); - appendixChunks.length = 0; - } - - if (x.mode === 'space' && changeToMdChunks.length) { - changeToMdChunks.length = 1; - } - if (changeToMdChunks.length) { - mdChunks.push(...changeToMdChunks); - } - } - - if (mdChunks.length) { - mdChunks[0] = mdChunks[0].trimStart(); - } - - return { meta: meta.info as Record, content: mdChunks.join(''), text: rawChunks.join('') }; + } } - async cachedExtract(url: string, cacheTolerance: number = 1000 * 3600 * 24, alternativeUrl?: string) { - if (!url) { - return undefined; + const mdChunks = []; + const appendixChunks = []; + mode = "space"; + for (const x of mdOps) { + const previousMode: string = mode; + const changeToMdChunks = []; + + const isNewStart = + x.mode !== "space" && + (x.op === "new" || + (previousMode === "appendix" && x.mode !== previousMode)); + + if (isNewStart) { + switch (x.mode) { + case "h1": { + changeToMdChunks.push(`\n\n# `); + mode = x.mode; + break; + } + + case "h2": { + changeToMdChunks.push(`\n\n## `); + mode = x.mode; + break; + } + + case "p": { + changeToMdChunks.push(`\n\n`); + mode = x.mode; + break; + } + + case "appendix": { + mode = x.mode; + appendixChunks.push(`\n\n`); + break; + } + + default: { + break; + } } - let nameUrl = alternativeUrl || url; - const digest = md5Hasher.hash(nameUrl); - - if (this.isDataUrl(url)) { - nameUrl = `blob://pdf:${digest}`; + } else { + if (x.mode === "appendix" && appendixChunks.length) { + const lastChunk = appendixChunks[appendixChunks.length - 1]; + if (!lastChunk.match(/(\s+|-)$/) && lastChunk.length !== 1) { + appendixChunks.push(" "); + } + } else if (mdChunks.length) { + const lastChunk = mdChunks[mdChunks.length - 1]; + if (!lastChunk.match(/(\s+|-)$/) && lastChunk.length !== 1) { + changeToMdChunks.push(" "); + } } + } - const cache: PDFContent | undefined = nameUrl.startsWith('blob:') ? undefined : - (await PDFContent.fromFirestoreQuery(PDFContent.COLLECTION.where('urlDigest', '==', digest).orderBy('createdAt', 'desc').limit(1)))?.[0]; - - if (cache) { - const age = Date.now() - cache?.createdAt.valueOf(); - const stale = cache.createdAt.valueOf() < (Date.now() - cacheTolerance); - this.logger.info(`${stale ? 'Stale cache exists' : 'Cache hit'} for PDF ${nameUrl}, normalized digest: ${digest}, ${age}ms old, tolerance ${cacheTolerance}ms`, { - data: url, url: nameUrl, digest, age, stale, cacheTolerance - }); - - if (!stale) { - if (cache.content && cache.text) { - return { - meta: cache.meta, - content: cache.content, - text: cache.text - }; - } - - try { - const r = await this.firebaseObjectStorage.downloadFile(`pdfs/${cache._id}`); - let cached = JSON.parse(r.toString('utf-8')); - - return { - meta: cached.meta, - content: cached.content, - text: cached.text - }; - } catch (err) { - this.logger.warn(`Unable to load cached content for ${nameUrl}`, { err }); - - return undefined; - } - } + if (x.text) { + if (x.mode == "appendix") { + if (appendixChunks.length || isNewStart) { + appendixChunks.push(x.text); + } else { + changeToMdChunks.push(x.text); + } + } else { + changeToMdChunks.push(x.text); } + } - let extracted; + if (isNewStart && x.mode !== "appendix" && appendixChunks.length) { + const appendix = appendixChunks + .join("") + .split(/\r?\n/) + .map((x) => x.trim()) + .filter(Boolean) + .map((x) => `> ${x}`) + .join("\n"); + changeToMdChunks.unshift(appendix); + changeToMdChunks.unshift(`\n\n`); + appendixChunks.length = 0; + } + + if (x.mode === "space" && changeToMdChunks.length) { + changeToMdChunks.length = 1; + } + if (changeToMdChunks.length) { + mdChunks.push(...changeToMdChunks); + } + } + + if (mdChunks.length) { + mdChunks[0] = mdChunks[0].trimStart(); + } + + return { + meta: meta.info as Record, + content: mdChunks.join(""), + text: rawChunks.join(""), + }; + } + + async cachedExtract( + url: string, + cacheTolerance: number = 1000 * 3600 * 24, + alternativeUrl?: string, + ) { + if (!url) { + return undefined; + } + let nameUrl = alternativeUrl || url; + const digest = md5Hasher.hash(nameUrl); + + if (this.isDataUrl(url)) { + nameUrl = `blob://pdf:${digest}`; + } + + const cache: PDFContent | undefined = nameUrl.startsWith("blob:") + ? undefined + : ( + await PDFContent.fromFirestoreQuery( + PDFContent.COLLECTION.where("urlDigest", "==", digest) + .orderBy("createdAt", "desc") + .limit(1), + ) + )?.[0]; + + if (cache) { + const age = Date.now() - cache?.createdAt.valueOf(); + const stale = cache.createdAt.valueOf() < Date.now() - cacheTolerance; + this.logger.info( + `${stale ? "Stale cache exists" : "Cache hit"} for PDF ${nameUrl}, normalized digest: ${digest}, ${age}ms old, tolerance ${cacheTolerance}ms`, + { + data: url, + url: nameUrl, + digest, + age, + stale, + cacheTolerance, + }, + ); + + if (!stale) { + if (cache.content && cache.text) { + return { + meta: cache.meta, + content: cache.content, + text: cache.text, + }; + } try { - extracted = await this.extract(url); - } catch (err: any) { - this.logger.warn(`Unable to extract from pdf ${nameUrl}`, { err, url, nameUrl }); - throw new AssertionFailureError(`Unable to process ${nameUrl} as pdf: ${err?.message}`); - } + const r = await this.firebaseObjectStorage.downloadFile( + `pdfs/${cache._id}`, + ); + let cached = JSON.parse(r.toString("utf-8")); - if (!this.asyncLocalContext.ctx.DNT && !nameUrl.startsWith('blob:')) { - const theID = randomUUID(); - await this.firebaseObjectStorage.saveFile(`pdfs/${theID}`, - Buffer.from(JSON.stringify(extracted), 'utf-8'), { contentType: 'application/json' }); - PDFContent.save( - PDFContent.from({ - _id: theID, - src: nameUrl, - meta: extracted?.meta || {}, - urlDigest: digest, - createdAt: new Date(), - expireAt: new Date(Date.now() + this.cacheRetentionMs) - }).degradeForFireStore() - ).catch((r) => { - this.logger.warn(`Unable to cache PDF content for ${nameUrl}`, { err: r }); - }); - } + return { + meta: cached.meta, + content: cached.content, + text: cached.text, + }; + } catch (err) { + this.logger.warn(`Unable to load cached content for ${nameUrl}`, { + err, + }); - return extracted; + return undefined; + } + } } - parsePdfDate(pdfDate: string | undefined) { - if (!pdfDate) { - return undefined; - } - // Remove the 'D:' prefix - const cleanedDate = pdfDate.slice(2); + let extracted; - // Define the format without the timezone part first - const dateTimePart = cleanedDate.slice(0, 14); - const timezonePart = cleanedDate.slice(14); - - // Construct the full date string in a standard format - const formattedDate = `${dateTimePart}${timezonePart.replace("'", "").replace("'", "")}`; - - // Parse the date with timezone - const parsedDate = dayjs(formattedDate, "YYYYMMDDHHmmssZ"); - - const date = parsedDate.toDate(); - - if (!date.valueOf()) { - return undefined; - } - - return date; + try { + extracted = await this.extract(url); + } catch (err: any) { + this.logger.warn(`Unable to extract from pdf ${nameUrl}`, { + err, + url, + nameUrl, + }); + throw new AssertionFailureError( + `Unable to process ${nameUrl} as pdf: ${err?.message}`, + ); } + + if (!this.asyncLocalContext.ctx.DNT && !nameUrl.startsWith("blob:")) { + const theID = randomUUID(); + await this.firebaseObjectStorage.saveFile( + `pdfs/${theID}`, + Buffer.from(JSON.stringify(extracted), "utf-8"), + { contentType: "application/json" }, + ); + PDFContent.save( + PDFContent.from({ + _id: theID, + src: nameUrl, + meta: extracted?.meta || {}, + urlDigest: digest, + createdAt: new Date(), + expireAt: new Date(Date.now() + this.cacheRetentionMs), + }).degradeForFireStore(), + ).catch((r) => { + this.logger.warn(`Unable to cache PDF content for ${nameUrl}`, { + err: r, + }); + }); + } + + return extracted; + } + + parsePdfDate(pdfDate: string | undefined) { + if (!pdfDate) { + return undefined; + } + // Remove the 'D:' prefix + const cleanedDate = pdfDate.slice(2); + + // Define the format without the timezone part first + const dateTimePart = cleanedDate.slice(0, 14); + const timezonePart = cleanedDate.slice(14); + + // Construct the full date string in a standard format + const formattedDate = `${dateTimePart}${timezonePart.replace("'", "").replace("'", "")}`; + + // Parse the date with timezone + const parsedDate = dayjs(formattedDate, "YYYYMMDDHHmmssZ"); + + const date = parsedDate.toDate(); + + if (!date.valueOf()) { + return undefined; + } + + return date; + } } diff --git a/src/services/pseudo-transfer.ts b/src/services/pseudo-transfer.ts index 45a2c76..a812d03 100644 --- a/src/services/pseudo-transfer.ts +++ b/src/services/pseudo-transfer.ts @@ -1,65 +1,67 @@ -import { marshalErrorLike } from 'civkit'; -import { AbstractPseudoTransfer, SYM_PSEUDO_TRANSFERABLE } from 'civkit/pseudo-transfer'; -import { container, singleton } from 'tsyringe'; - +import { marshalErrorLike } from "civkit"; +import { + AbstractPseudoTransfer, + SYM_PSEUDO_TRANSFERABLE, +} from "civkit/pseudo-transfer"; +import { container, singleton } from "tsyringe"; @singleton() export class PseudoTransfer extends AbstractPseudoTransfer { - - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } - + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } } const instance = container.resolve(PseudoTransfer); Object.defineProperty(Error.prototype, SYM_PSEUDO_TRANSFERABLE, { - value: function () { - const prototype = this; - return { - copyOwnProperty: 'all', - marshall: (input: Error) => marshalErrorLike(input), - unMarshall: (input: object) => { - Object.setPrototypeOf(input, prototype); - return input; - }, - }; - }, - enumerable: false, + value: function () { + const prototype = this; + return { + copyOwnProperty: "all", + marshall: (input: Error) => marshalErrorLike(input), + unMarshall: (input: object) => { + Object.setPrototypeOf(input, prototype); + return input; + }, + }; + }, + enumerable: false, }); instance.expectPseudoTransferableType(Error); -for (const x of [...Object.values(require('./errors')), ...Object.values(require('civkit/civ-rpc'))]) { - if (typeof x === 'function' && x.prototype instanceof Error) { - instance.expectPseudoTransferableType(x as any); - } +for (const x of [ + ...Object.values(require("./errors")), + ...Object.values(require("civkit/civ-rpc")), +]) { + if (typeof x === "function" && x.prototype instanceof Error) { + instance.expectPseudoTransferableType(x as any); + } } - Object.defineProperty(URL.prototype, SYM_PSEUDO_TRANSFERABLE, { - value: function () { - return { - copyOwnProperty: 'none', - marshall: (input: URL) => ({ href: input.href }), - unMarshall: (input: { href: string; }) => new URL(input.href), - }; - }, - enumerable: false, + value: function () { + return { + copyOwnProperty: "none", + marshall: (input: URL) => ({ href: input.href }), + unMarshall: (input: { href: string }) => new URL(input.href), + }; + }, + enumerable: false, }); instance.expectPseudoTransferableType(URL); Object.defineProperty(Buffer.prototype, SYM_PSEUDO_TRANSFERABLE, { - value: function () { - return { - copyOwnProperty: 'none', - unMarshall: (input: Uint8Array | Buffer) => Buffer.isBuffer(input) ? input : Buffer.from(input), - marshall: (input: Uint8Array | Buffer) => input, - }; - }, - enumerable: false, + value: function () { + return { + copyOwnProperty: "none", + unMarshall: (input: Uint8Array | Buffer) => + Buffer.isBuffer(input) ? input : Buffer.from(input), + marshall: (input: Uint8Array | Buffer) => input, + }; + }, + enumerable: false, }); instance.expectPseudoTransferableType(Buffer); - export default instance; diff --git a/src/services/puppeteer.ts b/src/services/puppeteer.ts index fd726f2..bddbd04 100644 --- a/src/services/puppeteer.ts +++ b/src/services/puppeteer.ts @@ -1,126 +1,142 @@ -import _ from 'lodash'; -import { isIP } from 'net'; -import { readFile } from 'fs/promises'; -import fs from 'fs'; -import { container, singleton } from 'tsyringe'; +import _ from "lodash"; +import { isIP } from "net"; +import { readFile } from "fs/promises"; +import fs from "fs"; +import { container, singleton } from "tsyringe"; -import type { Browser, CookieParam, GoToOptions, HTTPRequest, HTTPResponse, Page, Viewport } from 'puppeteer'; -import type { Cookie } from 'set-cookie-parser'; -import puppeteer, { TimeoutError } from 'puppeteer'; +import type { + Browser, + CookieParam, + GoToOptions, + HTTPRequest, + HTTPResponse, + Page, + Viewport, +} from "puppeteer"; +import type { Cookie } from "set-cookie-parser"; +import puppeteer, { TimeoutError } from "puppeteer"; -import { Defer, Deferred } from 'civkit/defer'; -import { AssertionFailureError, ParamValidationError } from 'civkit/civ-rpc'; -import { AsyncService } from 'civkit/async-service'; -import { FancyFile } from 'civkit/fancy-file'; -import { delay } from 'civkit/timeout'; +import { Defer, Deferred } from "civkit/defer"; +import { AssertionFailureError, ParamValidationError } from "civkit/civ-rpc"; +import { AsyncService } from "civkit/async-service"; +import { FancyFile } from "civkit/fancy-file"; +import { delay } from "civkit/timeout"; -import { SecurityCompromiseError, ServiceCrashedError, ServiceNodeResourceDrainError } from '../shared/lib/errors'; -import { CurlControl } from './curl'; -import { BlackHoleDetector } from './blackhole-detector'; -import { AsyncLocalContext } from './async-context'; -import { GlobalLogger } from './logger'; -import { minimalStealth } from './minimal-stealth'; -const tldExtract = require('tld-extract'); +import { + SecurityCompromiseError, + ServiceCrashedError, + ServiceNodeResourceDrainError, +} from "../shared/lib/errors"; +import { CurlControl } from "./curl"; +import { BlackHoleDetector } from "./blackhole-detector"; +import { AsyncLocalContext } from "./async-context"; +import { GlobalLogger } from "./logger"; +import { minimalStealth } from "./minimal-stealth"; +const tldExtract = require("tld-extract"); -const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8'); +const READABILITY_JS = fs.readFileSync( + require.resolve("@mozilla/readability/Readability.js"), + "utf-8", +); function getChromeLaunchArgs() { - const args = [ - '--disable-dev-shm-usage', - '--disable-blink-features=AutomationControlled', - ]; + const args = [ + "--disable-dev-shm-usage", + "--disable-blink-features=AutomationControlled", + ]; - // Docker and many CI kernels do not permit Chrome's sandbox namespaces. - if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) { - args.push('--no-sandbox', '--disable-setuid-sandbox'); - } + // Docker and many CI kernels do not permit Chrome's sandbox namespaces. + if ( + process.env.PUPPETEER_DISABLE_SANDBOX !== "false" && + fs.existsSync("/.dockerenv") + ) { + args.push("--no-sandbox", "--disable-setuid-sandbox"); + } - return args; + return args; } - export interface ImgBrief { - src: string; - loaded?: boolean; - width?: number; - height?: number; - naturalWidth?: number; - naturalHeight?: number; - alt?: string; + src: string; + loaded?: boolean; + width?: number; + height?: number; + naturalWidth?: number; + naturalHeight?: number; + alt?: string; } export interface ReadabilityParsed { - title: string; - content: string; - textContent: string; - length: number; - excerpt: string; - byline: string; - dir: string; - siteName: string; - lang: string; - publishedTime: string; + title: string; + content: string; + textContent: string; + length: number; + excerpt: string; + byline: string; + dir: string; + siteName: string; + lang: string; + publishedTime: string; } export interface PageSnapshot { - title: string; - description?: string; - href: string; - rebase?: string; - html: string; - htmlSignificantlyModifiedByJs?: boolean; - shadowExpanded?: string; - text: string; - status?: number; - statusText?: string; - parsed?: Partial | null; - screenshot?: Buffer; - pageshot?: Buffer; - imgs?: ImgBrief[]; - pdfs?: string[]; - maxElemDepth?: number; - elemCount?: number; - childFrames?: PageSnapshot[]; - isIntermediate?: boolean; - isFromCache?: boolean; - lastMutationIdle?: number; - lastContentResourceLoaded?: number; - lastMediaResourceLoaded?: number; + title: string; + description?: string; + href: string; + rebase?: string; + html: string; + htmlSignificantlyModifiedByJs?: boolean; + shadowExpanded?: string; + text: string; + status?: number; + statusText?: string; + parsed?: Partial | null; + screenshot?: Buffer; + pageshot?: Buffer; + imgs?: ImgBrief[]; + pdfs?: string[]; + maxElemDepth?: number; + elemCount?: number; + childFrames?: PageSnapshot[]; + isIntermediate?: boolean; + isFromCache?: boolean; + lastMutationIdle?: number; + lastContentResourceLoaded?: number; + lastMediaResourceLoaded?: number; } export interface ExtendedSnapshot extends PageSnapshot { - links: [string, string][]; - imgs: ImgBrief[]; + links: [string, string][]; + imgs: ImgBrief[]; } export interface ScrappingOptions { - proxyUrl?: string; - cookies?: Cookie[]; - favorScreenshot?: boolean; - waitForSelector?: string | string[]; - minIntervalMs?: number; - overrideUserAgent?: string; - timeoutMs?: number; - locale?: string; - referer?: string; - extraHeaders?: Record; - injectFrameScripts?: string[]; - injectPageScripts?: string[]; - viewport?: Viewport; - proxyResources?: boolean; + proxyUrl?: string; + cookies?: Cookie[]; + favorScreenshot?: boolean; + waitForSelector?: string | string[]; + minIntervalMs?: number; + overrideUserAgent?: string; + timeoutMs?: number; + locale?: string; + referer?: string; + extraHeaders?: Record; + injectFrameScripts?: string[]; + injectPageScripts?: string[]; + viewport?: Viewport; + proxyResources?: boolean; - sideLoad?: { - impersonate: { - [url: string]: { - status: number; - headers: { [k: string]: string | string[]; }; - contentType?: string; - body?: FancyFile; - }; - }; - proxyOrigin: { [origin: string]: string; }; + sideLoad?: { + impersonate: { + [url: string]: { + status: number; + headers: { [k: string]: string | string[] }; + contentType?: string; + body?: FancyFile; + }; }; - + proxyOrigin: { [origin: string]: string }; + }; } const SIMULATE_SCROLL = ` @@ -455,294 +471,340 @@ window.briefImgs = briefImgs; `; const documentResourceTypes = new Set([ - 'document', 'script', 'xhr', 'fetch', 'prefetch', 'eventsource', 'websocket', 'preflight' + "document", + "script", + "xhr", + "fetch", + "prefetch", + "eventsource", + "websocket", + "preflight", ]); -const mediaResourceTypes = new Set([ - 'stylesheet', 'image', 'font', 'media' -]); - +const mediaResourceTypes = new Set(["stylesheet", "image", "font", "media"]); class PageReqCtrlKit { - reqSet: Set = new Set(); - blockers: Deferred[] = []; - lastResourceLoadedAt: number = 0; - lastContentResourceLoadedAt: number = 0; - lastMediaResourceLoadedAt: number = 0; + reqSet: Set = new Set(); + blockers: Deferred[] = []; + lastResourceLoadedAt: number = 0; + lastContentResourceLoadedAt: number = 0; + lastMediaResourceLoadedAt: number = 0; - constructor( - public concurrency: number, - ) { - if (isNaN(concurrency) || concurrency < 1) { - throw new AssertionFailureError(`Invalid concurrency: ${concurrency}`); - } + constructor(public concurrency: number) { + if (isNaN(concurrency) || concurrency < 1) { + throw new AssertionFailureError(`Invalid concurrency: ${concurrency}`); } + } - onNewRequest(req: HTTPRequest) { - this.reqSet.add(req); - if (this.reqSet.size <= this.concurrency) { - return Promise.resolve(); - } - const deferred = Defer(); - this.blockers.push(deferred); - - return deferred.promise; + onNewRequest(req: HTTPRequest) { + this.reqSet.add(req); + if (this.reqSet.size <= this.concurrency) { + return Promise.resolve(); } + const deferred = Defer(); + this.blockers.push(deferred); - onFinishRequest(req: HTTPRequest) { - this.reqSet.delete(req); - const deferred = this.blockers.shift(); - deferred?.resolve(); - const now = Date.now(); - this.lastResourceLoadedAt = now; - // Beware req being undefined - // https://pptr.dev/api/puppeteer.pageevent#:~:text=For%20certain%20requests%2C%20might%20contain%20undefined. - const typ = req?.resourceType(); - if (!typ) { - return; - } - if (documentResourceTypes.has(typ)) { - this.lastContentResourceLoadedAt = now; - } - if (mediaResourceTypes.has(typ)) { - this.lastMediaResourceLoadedAt = now; - } + return deferred.promise; + } + + onFinishRequest(req: HTTPRequest) { + this.reqSet.delete(req); + const deferred = this.blockers.shift(); + deferred?.resolve(); + const now = Date.now(); + this.lastResourceLoadedAt = now; + // Beware req being undefined + // https://pptr.dev/api/puppeteer.pageevent#:~:text=For%20certain%20requests%2C%20might%20contain%20undefined. + const typ = req?.resourceType(); + if (!typ) { + return; } + if (documentResourceTypes.has(typ)) { + this.lastContentResourceLoadedAt = now; + } + if (mediaResourceTypes.has(typ)) { + this.lastMediaResourceLoadedAt = now; + } + } } @singleton() export class PuppeteerControl extends AsyncService { + _sn = 0; + browser!: Browser; + logger = this.globalLogger.child({ service: this.constructor.name }); - _sn = 0; - browser!: Browser; - logger = this.globalLogger.child({ service: this.constructor.name }); + __loadedPage: Page[] = []; - __loadedPage: Page[] = []; + finalizerMap = new WeakMap>(); + snMap = new WeakMap(); + livePages = new Set(); + pagePhase = new WeakMap(); + lastPageCratedAt: number = 0; + ua: string = ""; + effectiveUA: string = ""; - finalizerMap = new WeakMap>(); - snMap = new WeakMap(); - livePages = new Set(); - pagePhase = new WeakMap(); - lastPageCratedAt: number = 0; - ua: string = ''; - effectiveUA: string = ''; + concurrentRequestsPerPage: number = 32; + pageReqCtrl = new WeakMap(); - concurrentRequestsPerPage: number = 32; - pageReqCtrl = new WeakMap(); + lastReqSentAt: number = 0; - lastReqSentAt: number = 0; + circuitBreakerHosts: Set = new Set(); - circuitBreakerHosts: Set = new Set(); + lifeCycleTrack = new WeakMap(); - lifeCycleTrack = new WeakMap(); + constructor( + protected globalLogger: GlobalLogger, + protected asyncLocalContext: AsyncLocalContext, + protected curlControl: CurlControl, + protected blackHoleDetector: BlackHoleDetector, + ) { + super(...arguments); + this.setMaxListeners(Infinity); - constructor( - protected globalLogger: GlobalLogger, - protected asyncLocalContext: AsyncLocalContext, - protected curlControl: CurlControl, - protected blackHoleDetector: BlackHoleDetector, - ) { - super(...arguments); - this.setMaxListeners(Infinity); - - let crippledTimes = 0; - this.on('crippled', () => { - crippledTimes += 1; - this.__loadedPage.length = 0; - this.livePages.clear(); - if (crippledTimes > 5) { - process.nextTick(() => { - this.emit('error', new Error('Browser crashed too many times, quitting...')); - // process.exit(1); - }); - } + let crippledTimes = 0; + this.on("crippled", () => { + crippledTimes += 1; + this.__loadedPage.length = 0; + this.livePages.clear(); + if (crippledTimes > 5) { + process.nextTick(() => { + this.emit( + "error", + new Error("Browser crashed too many times, quitting..."), + ); + // process.exit(1); }); + } + }); + } + + override async init() { + await this.dependencyReady(); + if (process.env.NODE_ENV?.includes("dry-run")) { + this.emit("ready"); + return; } - override async init() { - await this.dependencyReady(); - if (process.env.NODE_ENV?.includes('dry-run')) { - this.emit('ready'); - return; - } - - if (this.browser) { - if (this.browser.connected) { - await this.browser.close(); - } else { - this.browser.process()?.kill('SIGKILL'); - } - } - this.browser = await puppeteer.launch({ - timeout: 30_000, - headless: !Boolean(process.env.DEBUG_BROWSER), - pipe: true, - executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH, - args: getChromeLaunchArgs(), - }).catch((err: any) => { - this.logger.error(`Unknown firebase issue, just die fast.`, { err }); - process.nextTick(() => { - this.emit('error', err); - // process.exit(1); - }); - return Promise.reject(err); + if (this.browser) { + if (this.browser.connected) { + await this.browser.close(); + } else { + this.browser.process()?.kill("SIGKILL"); + } + } + this.browser = await puppeteer + .launch({ + timeout: 30_000, + headless: !Boolean(process.env.DEBUG_BROWSER), + pipe: true, + executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH, + args: getChromeLaunchArgs(), + }) + .catch((err: any) => { + this.logger.error(`Unknown firebase issue, just die fast.`, { err }); + process.nextTick(() => { + this.emit("error", err); + // process.exit(1); }); - this.browser.once('disconnected', () => { - this.logger.warn(`Browser disconnected`); - if (this.browser) { - this.emit('crippled'); - } - process.nextTick(() => this.serviceReady()); - }); - this.ua = await this.browser.userAgent(); - this.logger.info(`Browser launched: ${this.browser.process()?.pid}, ${this.ua}`); - this.effectiveUA = this.ua.replace(/Headless/i, '').replace('Mozilla/5.0 (X11; Linux x86_64)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'); - this.curlControl.impersonateChrome(this.effectiveUA); + return Promise.reject(err); + }); + this.browser.once("disconnected", () => { + this.logger.warn(`Browser disconnected`); + if (this.browser) { + this.emit("crippled"); + } + process.nextTick(() => this.serviceReady()); + }); + this.ua = await this.browser.userAgent(); + this.logger.info( + `Browser launched: ${this.browser.process()?.pid}, ${this.ua}`, + ); + this.effectiveUA = this.ua + .replace(/Headless/i, "") + .replace( + "Mozilla/5.0 (X11; Linux x86_64)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + ); + this.curlControl.impersonateChrome(this.effectiveUA); - await this.newPage('beware_deadlock').then((r) => this.__loadedPage.push(r)); + await this.newPage("beware_deadlock").then((r) => + this.__loadedPage.push(r), + ); - this.emit('ready'); + this.emit("ready"); + } + + protected getRpsControlKit(page: Page) { + let kit = this.pageReqCtrl.get(page); + if (!kit) { + kit = new PageReqCtrlKit(this.concurrentRequestsPerPage); + this.pageReqCtrl.set(page, kit); } - protected getRpsControlKit(page: Page) { - let kit = this.pageReqCtrl.get(page); - if (!kit) { - kit = new PageReqCtrlKit(this.concurrentRequestsPerPage); - this.pageReqCtrl.set(page, kit); - } + return kit; + } - return kit; + async newPage(bewareDeadLock: any = false) { + if (!bewareDeadLock) { + await this.serviceReady(); } + const sn = this._sn++; + let page; + try { + const dedicatedContext = await this.browser.createBrowserContext(); + page = await dedicatedContext.newPage(); + } catch (err: any) { + this.logger.warn(`Failed to create page ${sn}`, { err }); + this.browser.process()?.kill("SIGKILL"); + throw new ServiceNodeResourceDrainError( + `This specific worker node failed to open a new page, try again.`, + ); + } + const preparations = []; - async newPage(bewareDeadLock: any = false) { - if (!bewareDeadLock) { - await this.serviceReady(); + preparations.push(page.setUserAgent(this.effectiveUA)); + // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); + // preparations.push(page.setUserAgent(`Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`)); + preparations.push(page.setBypassCSP(true)); + preparations.push(page.setViewport({ width: 1024, height: 1024 })); + preparations.push( + page.exposeFunction("reportSnapshot", (snapshot: PageSnapshot) => { + if (snapshot.href === "about:blank") { + return; } - const sn = this._sn++; - let page; + page.emit("snapshot", snapshot); + }), + ); + preparations.push( + page.exposeFunction("setViewport", (viewport: Viewport | null) => { + page.setViewport(viewport).catch(() => undefined); + }), + ); + preparations.push(page.evaluateOnNewDocument(SCRIPT_TO_INJECT_INTO_FRAME)); + preparations.push(page.setRequestInterception(true)); + + await Promise.all(preparations); + + await page.goto("about:blank", { waitUntil: "domcontentloaded" }); + + const domainSet = new Set(); + let reqCounter = 0; + let t0: number | undefined; + let halt = false; + + page.on("request", async (req) => { + reqCounter++; + if (halt) { + return req.abort("blockedbyclient", 1000); + } + const requestUrl = req.url(); + if ( + !requestUrl.startsWith("http:") && + !requestUrl.startsWith("https:") && + !requestUrl.startsWith("chrome-extension:") && + requestUrl !== "about:blank" + ) { + return req.abort("blockedbyclient", 1000); + } + t0 ??= Date.now(); + + const parsedUrl = new URL(requestUrl); + if (isIP(parsedUrl.hostname)) { + domainSet.add(parsedUrl.hostname); + } else { try { - const dedicatedContext = await this.browser.createBrowserContext(); - page = await dedicatedContext.newPage(); - } catch (err: any) { - this.logger.warn(`Failed to create page ${sn}`, { err }); - this.browser.process()?.kill('SIGKILL'); - throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`); + const tldParsed = tldExtract(requestUrl); + domainSet.add(tldParsed.domain); + } catch (_err) { + domainSet.add(parsedUrl.hostname); } - const preparations = []; + } - preparations.push(page.setUserAgent(this.effectiveUA)); - // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); - // preparations.push(page.setUserAgent(`Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`)); - preparations.push(page.setBypassCSP(true)); - preparations.push(page.setViewport({ width: 1024, height: 1024 })); - preparations.push(page.exposeFunction('reportSnapshot', (snapshot: PageSnapshot) => { - if (snapshot.href === 'about:blank') { - return; - } - page.emit('snapshot', snapshot); - })); - preparations.push(page.exposeFunction('setViewport', (viewport: Viewport | null) => { - page.setViewport(viewport).catch(() => undefined); - })); - preparations.push(page.evaluateOnNewDocument(SCRIPT_TO_INJECT_INTO_FRAME)); - preparations.push(page.setRequestInterception(true)); - - await Promise.all(preparations); - - await page.goto('about:blank', { waitUntil: 'domcontentloaded' }); - - const domainSet = new Set(); - let reqCounter = 0; - let t0: number | undefined; - let halt = false; - - page.on('request', async (req) => { - reqCounter++; - if (halt) { - return req.abort('blockedbyclient', 1000); - } - const requestUrl = req.url(); - if (!requestUrl.startsWith('http:') && !requestUrl.startsWith('https:') && !requestUrl.startsWith('chrome-extension:') && requestUrl !== 'about:blank') { - return req.abort('blockedbyclient', 1000); - } - t0 ??= Date.now(); - - const parsedUrl = new URL(requestUrl); - if (isIP(parsedUrl.hostname)) { - domainSet.add(parsedUrl.hostname); - } else { - try { - const tldParsed = tldExtract(requestUrl); - domainSet.add(tldParsed.domain); - } catch (_err) { - domainSet.add(parsedUrl.hostname); - } - } - - if (this.circuitBreakerHosts.has(parsedUrl.hostname.toLowerCase())) { - page.emit('abuse', { url: requestUrl, page, sn, reason: `Abusive request: ${requestUrl}` }); - return req.abort('blockedbyclient', 1000); - } - - if ( - parsedUrl.hostname === 'localhost' || - parsedUrl.hostname.startsWith('127.') - ) { - page.emit('abuse', { url: requestUrl, page, sn, reason: `Suspicious action: Request to localhost: ${requestUrl}` }); - - return req.abort('blockedbyclient', 1000); - } - - const dt = Math.ceil((Date.now() - t0) / 1000); - const rps = reqCounter / dt; - // console.log(`rps: ${rps}`); - const pagePhase = this.pagePhase.get(page); - if (pagePhase === 'background') { - if (rps > 10 || reqCounter > 1000) { - halt = true; - - return req.abort('blockedbyclient', 1000); - } - } - if (reqCounter > 1000) { - if (rps > 60 || reqCounter > 2000) { - page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many requests` }); - halt = true; - - return req.abort('blockedbyclient', 1000); - } - } - - if (domainSet.size > 200) { - page.emit('abuse', { url: requestUrl, page, sn, reason: `DDoS attack suspected: Too many domains` }); - halt = true; - - return req.abort('blockedbyclient', 1000); - } - - if (requestUrl.startsWith('http')) { - const kit = this.getRpsControlKit(page); - await kit.onNewRequest(req); - } - - if (req.isInterceptResolutionHandled()) { - return; - }; - - const continueArgs = req.continueRequestOverrides - ? [req.continueRequestOverrides(), 0] as const - : []; - - return req.continue(continueArgs[0], continueArgs[1]); + if (this.circuitBreakerHosts.has(parsedUrl.hostname.toLowerCase())) { + page.emit("abuse", { + url: requestUrl, + page, + sn, + reason: `Abusive request: ${requestUrl}`, }); - const reqFinishHandler = (req: HTTPRequest) => { - const kit = this.getRpsControlKit(page); - kit.onFinishRequest(req); - }; - page.on('requestfinished', reqFinishHandler); - page.on('requestfailed', reqFinishHandler); - page.on('requestservedfromcache', reqFinishHandler); + return req.abort("blockedbyclient", 1000); + } - await page.evaluateOnNewDocument(` + if ( + parsedUrl.hostname === "localhost" || + parsedUrl.hostname.startsWith("127.") + ) { + page.emit("abuse", { + url: requestUrl, + page, + sn, + reason: `Suspicious action: Request to localhost: ${requestUrl}`, + }); + + return req.abort("blockedbyclient", 1000); + } + + const dt = Math.ceil((Date.now() - t0) / 1000); + const rps = reqCounter / dt; + // console.log(`rps: ${rps}`); + const pagePhase = this.pagePhase.get(page); + if (pagePhase === "background") { + if (rps > 10 || reqCounter > 1000) { + halt = true; + + return req.abort("blockedbyclient", 1000); + } + } + if (reqCounter > 1000) { + if (rps > 60 || reqCounter > 2000) { + page.emit("abuse", { + url: requestUrl, + page, + sn, + reason: `DDoS attack suspected: Too many requests`, + }); + halt = true; + + return req.abort("blockedbyclient", 1000); + } + } + + if (domainSet.size > 200) { + page.emit("abuse", { + url: requestUrl, + page, + sn, + reason: `DDoS attack suspected: Too many domains`, + }); + halt = true; + + return req.abort("blockedbyclient", 1000); + } + + if (requestUrl.startsWith("http")) { + const kit = this.getRpsControlKit(page); + await kit.onNewRequest(req); + } + + if (req.isInterceptResolutionHandled()) { + return; + } + + const continueArgs = req.continueRequestOverrides + ? ([req.continueRequestOverrides(), 0] as const) + : []; + + return req.continue(continueArgs[0], continueArgs[1]); + }); + const reqFinishHandler = (req: HTTPRequest) => { + const kit = this.getRpsControlKit(page); + kit.onFinishRequest(req); + }; + page.on("requestfinished", reqFinishHandler); + page.on("requestfailed", reqFinishHandler); + page.on("requestservedfromcache", reqFinishHandler); + + await page.evaluateOnNewDocument(` (function () { if (window.self === window.top) { let lastAnalytics; @@ -790,556 +852,667 @@ export class PuppeteerControl extends AsyncService { })(); `); - this.snMap.set(page, sn); - this.logger.debug(`Page ${sn} created.`); - this.lastPageCratedAt = Date.now(); - this.livePages.add(page); - this.pagePhase.set(page, 'idle'); + this.snMap.set(page, sn); + this.logger.debug(`Page ${sn} created.`); + this.lastPageCratedAt = Date.now(); + this.livePages.add(page); + this.pagePhase.set(page, "idle"); - return page; - } + return page; + } - async getNextPage() { - let thePage: Page | undefined; - if (this.__loadedPage.length) { - thePage = this.__loadedPage.shift(); - if (this.__loadedPage.length <= 1) { - process.nextTick(() => { - this.newPage() - .then((r) => this.__loadedPage.push(r)) - .catch((err) => { - this.logger.warn(`Failed to load new page ahead of time`, { err }); - }); - }); - } - } - - if (!thePage) { - thePage = await this.newPage(); - } - - const timer = setTimeout(() => { - this.logger.warn(`Page is not allowed to live past 5 minutes, ditching page ${this.snMap.get(thePage!)}...`); - this.ditchPage(thePage!); - }, 300 * 1000); - - this.finalizerMap.set(thePage, timer); - - return thePage; - } - - async ditchPage(page: Page) { - if (this.finalizerMap.has(page)) { - clearTimeout(this.finalizerMap.get(page)!); - this.finalizerMap.delete(page); - } - if (page.isClosed()) { - return; - } - const sn = this.snMap.get(page); - this.logger.debug(`Closing page ${sn}`); - await Promise.race([ - (async () => { - const ctx = page.browserContext(); - try { - await page.close(); - } finally { - await ctx.close(); - } - })(), - delay(5000) - ]).catch((err) => { - this.logger.error(`Failed to destroy page ${sn}`, { err }); + async getNextPage() { + let thePage: Page | undefined; + if (this.__loadedPage.length) { + thePage = this.__loadedPage.shift(); + if (this.__loadedPage.length <= 1) { + process.nextTick(() => { + this.newPage() + .then((r) => this.__loadedPage.push(r)) + .catch((err) => { + this.logger.warn(`Failed to load new page ahead of time`, { + err, + }); + }); }); - this.livePages.delete(page); - this.pagePhase.delete(page); + } } - async *scrap(parsedUrl: URL, options: ScrappingOptions = {}): AsyncGenerator { - // parsedUrl.search = ''; - const url = parsedUrl.toString(); - let snapshot: PageSnapshot | undefined; - let screenshot: Buffer | undefined; - let pageshot: Buffer | undefined; - const pdfUrls: string[] = []; - let navigationResponse: HTTPResponse | undefined; - const page = await this.getNextPage(); - this.lifeCycleTrack.set(page, this.asyncLocalContext.ctx); - this.pagePhase.set(page, 'active'); - page.on('response', (resp) => { - this.blackHoleDetector.itWorked(); - const req = resp.request(); - if (req.frame() === page.mainFrame() && req.isNavigationRequest()) { - navigationResponse = resp; - } - if (!resp.ok()) { - return; - } - const headers = resp.headers(); - const url = resp.url(); - const contentType = headers['content-type']; - if (contentType?.toLowerCase().includes('application/pdf')) { - pdfUrls.push(url); - } - }); - page.on('request', async (req) => { - if (req.isInterceptResolutionHandled()) { - return; - }; - const reqUrlParsed = new URL(req.url()); - if (!reqUrlParsed.protocol.startsWith('http')) { - const overrides = req.continueRequestOverrides(); + if (!thePage) { + thePage = await this.newPage(); + } - return req.continue(overrides, 0); - } - const typ = req.resourceType(); - if (typ === 'media') { - // Non-cooperative answer to block all media requests. - return req.abort('blockedbyclient'); - } - if (!options.proxyResources) { - const isDocRequest = ['document', 'xhr', 'fetch', 'websocket', 'prefetch', 'eventsource', 'ping'].includes(typ); - if (!isDocRequest) { - if (options.extraHeaders) { - const overrides = req.continueRequestOverrides(); - const continueArgs = [{ - ...overrides, - headers: { - ...req.headers(), - ...overrides?.headers, - ...options.extraHeaders, - } - }, 1] as const; + const timer = setTimeout(() => { + this.logger.warn( + `Page is not allowed to live past 5 minutes, ditching page ${this.snMap.get(thePage!)}...`, + ); + this.ditchPage(thePage!); + }, 300 * 1000); - return req.continue(continueArgs[0], continueArgs[1]); - } - const overrides = req.continueRequestOverrides(); + this.finalizerMap.set(thePage, timer); - return req.continue(overrides, 0); - } - } - const sideload = options.sideLoad; + return thePage; + } - const impersonate = sideload?.impersonate[reqUrlParsed.href]; - if (impersonate) { - let body; - if (impersonate.body) { - body = await readFile(await impersonate.body.filePath); - if (req.isInterceptResolutionHandled()) { - return; - } - } - return req.respond({ - status: impersonate.status, - headers: impersonate.headers, - contentType: impersonate.contentType, - body: body ? Uint8Array.from(body) : undefined, - }, 999); - } + async ditchPage(page: Page) { + if (this.finalizerMap.has(page)) { + clearTimeout(this.finalizerMap.get(page)!); + this.finalizerMap.delete(page); + } + if (page.isClosed()) { + return; + } + const sn = this.snMap.get(page); + this.logger.debug(`Closing page ${sn}`); + await Promise.race([ + (async () => { + const ctx = page.browserContext(); + try { + await page.close(); + } finally { + await ctx.close(); + } + })(), + delay(5000), + ]).catch((err) => { + this.logger.error(`Failed to destroy page ${sn}`, { err }); + }); + this.livePages.delete(page); + this.pagePhase.delete(page); + } - const proxy = options.proxyUrl || sideload?.proxyOrigin?.[reqUrlParsed.origin]; - const ctx = this.lifeCycleTrack.get(page); - if (proxy && ctx) { - return await this.asyncLocalContext.bridge(ctx, async () => { - try { - const curled = await this.curlControl.sideLoad(reqUrlParsed, { - ...options, - method: req.method(), - body: req.postData(), - extraHeaders: { - ...req.headers(), - ...options.extraHeaders, - }, - proxyUrl: proxy - }); - if (req.isInterceptResolutionHandled()) { - return; - }; + async *scrap( + parsedUrl: URL, + options: ScrappingOptions = {}, + ): AsyncGenerator { + // parsedUrl.search = ''; + const url = parsedUrl.toString(); + let snapshot: PageSnapshot | undefined; + let screenshot: Buffer | undefined; + let pageshot: Buffer | undefined; + const pdfUrls: string[] = []; + let navigationResponse: HTTPResponse | undefined; + const page = await this.getNextPage(); + this.lifeCycleTrack.set(page, this.asyncLocalContext.ctx); + this.pagePhase.set(page, "active"); + page.on("response", (resp) => { + this.blackHoleDetector.itWorked(); + const req = resp.request(); + if (req.frame() === page.mainFrame() && req.isNavigationRequest()) { + navigationResponse = resp; + } + if (!resp.ok()) { + return; + } + const headers = resp.headers(); + const url = resp.url(); + const contentType = headers["content-type"]; + if (contentType?.toLowerCase().includes("application/pdf")) { + pdfUrls.push(url); + } + }); + page.on("request", async (req) => { + if (req.isInterceptResolutionHandled()) { + return; + } + const reqUrlParsed = new URL(req.url()); + if (!reqUrlParsed.protocol.startsWith("http")) { + const overrides = req.continueRequestOverrides(); - if (curled.chain.length === 1) { - if (!curled.file) { - return req.respond({ - status: curled.status, - headers: _.omit(curled.headers, 'result'), - contentType: curled.contentType, - }, 3); - } - const body = await readFile(await curled.file.filePath); - if (req.isInterceptResolutionHandled()) { - return; - }; - return req.respond({ - status: curled.status, - headers: _.omit(curled.headers, 'result'), - contentType: curled.contentType, - body: Uint8Array.from(body), - }, 3); - } - options.sideLoad ??= curled.sideLoadOpts; - _.merge(options.sideLoad, curled.sideLoadOpts); - const firstReq = curled.chain[0]; - - return req.respond({ - status: (firstReq.result as any)!.code, - headers: _.omit(firstReq, 'result'), - }, 3); - } catch (err: any) { - this.logger.warn(`Failed to sideload browser request ${reqUrlParsed.origin}`, { href: reqUrlParsed.href, err, proxy }); - } - if (req.isInterceptResolutionHandled()) { - return; - }; - const overrides = req.continueRequestOverrides(); - const continueArgs = [{ - ...overrides, - headers: { - ...req.headers(), - ...overrides?.headers, - ...options.extraHeaders, - } - }, 1] as const; - - return req.continue(continueArgs[0], continueArgs[1]); - }); - } - - if (req.isInterceptResolutionHandled()) { - return; - }; + return req.continue(overrides, 0); + } + const typ = req.resourceType(); + if (typ === "media") { + // Non-cooperative answer to block all media requests. + return req.abort("blockedbyclient"); + } + if (!options.proxyResources) { + const isDocRequest = [ + "document", + "xhr", + "fetch", + "websocket", + "prefetch", + "eventsource", + "ping", + ].includes(typ); + if (!isDocRequest) { + if (options.extraHeaders) { const overrides = req.continueRequestOverrides(); - const continueArgs = [{ + const continueArgs = [ + { ...overrides, headers: { - ...req.headers(), - ...overrides?.headers, - ...options.extraHeaders, - } - }, 1] as const; + ...req.headers(), + ...overrides?.headers, + ...options.extraHeaders, + }, + }, + 1, + ] as const; return req.continue(continueArgs[0], continueArgs[1]); - }); - let pageScriptEvaluations: Promise[] = []; - let frameScriptEvaluations: Promise[] = []; - if (options.injectPageScripts?.length) { - page.on('framenavigated', (frame) => { - if (frame !== page.mainFrame()) { - return; - } + } + const overrides = req.continueRequestOverrides(); - pageScriptEvaluations.push( - Promise.allSettled(options.injectPageScripts!.map((x) => frame.evaluate(x).catch((err) => { - this.logger.warn(`Error in evaluation of page scripts`, { err }); - }))) + return req.continue(overrides, 0); + } + } + const sideload = options.sideLoad; + + const impersonate = sideload?.impersonate[reqUrlParsed.href]; + if (impersonate) { + let body; + if (impersonate.body) { + body = await readFile(await impersonate.body.filePath); + if (req.isInterceptResolutionHandled()) { + return; + } + } + return req.respond( + { + status: impersonate.status, + headers: impersonate.headers, + contentType: impersonate.contentType, + body: body ? Uint8Array.from(body) : undefined, + }, + 999, + ); + } + + const proxy = + options.proxyUrl || sideload?.proxyOrigin?.[reqUrlParsed.origin]; + const ctx = this.lifeCycleTrack.get(page); + if (proxy && ctx) { + return await this.asyncLocalContext.bridge(ctx, async () => { + try { + const curled = await this.curlControl.sideLoad(reqUrlParsed, { + ...options, + method: req.method(), + body: req.postData(), + extraHeaders: { + ...req.headers(), + ...options.extraHeaders, + }, + proxyUrl: proxy, + }); + if (req.isInterceptResolutionHandled()) { + return; + } + + if (curled.chain.length === 1) { + if (!curled.file) { + return req.respond( + { + status: curled.status, + headers: _.omit(curled.headers, "result"), + contentType: curled.contentType, + }, + 3, ); - }); - } - if (options.injectFrameScripts?.length) { - page.on('framenavigated', (frame) => { - frameScriptEvaluations.push( - Promise.allSettled(options.injectFrameScripts!.map((x) => frame.evaluate(x).catch((err) => { - this.logger.warn(`Error in evaluation of frame scripts`, { err }); - }))) - ); - }); - } - const sn = this.snMap.get(page); - this.logger.info(`Page ${sn}: Scraping ${url}`, { url }); - if (options.locale) { - // Add headers via request interception to walk around this bug - // https://github.com/puppeteer/puppeteer/issues/10235 - // await page.setExtraHTTPHeaders({ - // 'Accept-Language': options.locale - // }); - - await page.evaluateOnNewDocument(() => { - Object.defineProperty(navigator, "language", { - get: function () { - return options.locale; - } - }); - Object.defineProperty(navigator, "languages", { - get: function () { - return [options.locale]; - } - }); - }); - } - - if (options.cookies) { - const mapped = options.cookies.map((x) => { - const draft: CookieParam = { - name: x.name, - value: encodeURIComponent(x.value), - secure: x.secure, - domain: x.domain, - path: x.path, - expires: x.expires ? Math.floor(x.expires.valueOf() / 1000) : undefined, - sameSite: x.sameSite as any, - }; - if (!draft.expires && x.maxAge) { - draft.expires = Math.floor(Date.now() / 1000) + x.maxAge; - } - if (!draft.domain) { - draft.url = parsedUrl.toString(); - } - - return draft; - }); - try { - await page.setCookie(...mapped); - } catch (err: any) { - this.logger.warn(`Page ${sn}: Failed to set cookies`, { err }); - throw new ParamValidationError({ - path: 'cookies', - message: `Failed to set cookies: ${err?.message}` - }); - } - } - if (options.overrideUserAgent) { - await page.setUserAgent(options.overrideUserAgent); - } - if (options.viewport) { - await page.setViewport(options.viewport); - } - - let nextSnapshotDeferred = Defer(); - const crippleListener = () => nextSnapshotDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` })); - this.once('crippled', crippleListener); - nextSnapshotDeferred.promise.finally(() => { - this.off('crippled', crippleListener); - }); - let successfullyDone; - const hdl = (s: any) => { - if (snapshot === s) { + } + const body = await readFile(await curled.file.filePath); + if (req.isInterceptResolutionHandled()) { return; - } - snapshot = s; - if (snapshot) { - const kit = this.pageReqCtrl.get(page); - snapshot.lastContentResourceLoaded = kit?.lastContentResourceLoadedAt; - snapshot.lastMediaResourceLoaded = kit?.lastMediaResourceLoadedAt; - } - if (s?.maxElemDepth && s.maxElemDepth > 256) { - return; - } - if (s?.elemCount && s.elemCount > 10_000) { - return; - } - nextSnapshotDeferred.resolve(s); - nextSnapshotDeferred = Defer(); - this.once('crippled', crippleListener); - nextSnapshotDeferred.promise.finally(() => { - this.off('crippled', crippleListener); - }); - }; - page.on('snapshot', hdl); - page.once('abuse', (event: any) => { - this.emit('abuse', { ...event, url: parsedUrl }); - if (snapshot?.href && parsedUrl.href !== snapshot.href) { - this.emit('abuse', { ...event, url: snapshot.href }); - } - - nextSnapshotDeferred.reject( - new SecurityCompromiseError(`Abuse detected: ${event.reason}`) - ); - }); - - const timeout = options.timeoutMs || 30_000; - const goToOptions: GoToOptions = { - waitUntil: ['load', 'domcontentloaded', 'networkidle0'], - timeout, - }; - - if (options.referer) { - goToOptions.referer = options.referer; - } - - let waitForPromise: Promise | undefined; - let finalizationPromise: Promise | undefined; - const doFinalization = async () => { - if (waitForPromise) { - // SuccessfullyDone is meant for the finish of the page. - // It doesn't matter if you are expecting something and it didn't show up. - await waitForPromise.catch(() => void 0); - } - successfullyDone ??= true; - try { - const pSubFrameSnapshots = this.snapshotChildFrames(page); - snapshot = await page.evaluate('giveSnapshot(true)') as PageSnapshot; - screenshot = (await this.takeScreenShot(page)) || screenshot; - pageshot = (await this.takeScreenShot(page, { fullPage: true })) || pageshot; - if (snapshot) { - snapshot.childFrames = await pSubFrameSnapshots; - } - } catch (err: any) { - this.logger.warn(`Page ${sn}: Failed to finalize ${url}`, { err }); - } - if (!snapshot?.html) { - return; - } - - this.logger.info(`Page ${sn}: Snapshot of ${url} done`, { url, title: snapshot?.title, href: snapshot?.href }); - this.emit( - 'crawled', + } + return req.respond( { - ...snapshot, - status: navigationResponse?.status(), - statusText: navigationResponse?.statusText(), - pdfs: _.uniq(pdfUrls), screenshot, pageshot, + status: curled.status, + headers: _.omit(curled.headers, "result"), + contentType: curled.contentType, + body: Uint8Array.from(body), }, - { ...options, url: parsedUrl } + 3, + ); + } + options.sideLoad ??= curled.sideLoadOpts; + _.merge(options.sideLoad, curled.sideLoadOpts); + const firstReq = curled.chain[0]; + + return req.respond( + { + status: (firstReq.result as any)!.code, + headers: _.omit(firstReq, "result"), + }, + 3, ); - }; - const delayPromise = delay(timeout); - const gotoPromise = page.goto(url, goToOptions) - .catch((err) => { - if (err instanceof TimeoutError) { - this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err }); - return new AssertionFailureError({ - message: `Failed to goto ${url}: ${err}`, - cause: err, - }); - } - if (err?.message?.startsWith('net::ERR_ABORTED')) { - if (pdfUrls.length) { - // Not throw for pdf mode. - return; - } - } + } catch (err: any) { + this.logger.warn( + `Failed to sideload browser request ${reqUrlParsed.origin}`, + { href: reqUrlParsed.href, err, proxy }, + ); + } + if (req.isInterceptResolutionHandled()) { + return; + } + const overrides = req.continueRequestOverrides(); + const continueArgs = [ + { + ...overrides, + headers: { + ...req.headers(), + ...overrides?.headers, + ...options.extraHeaders, + }, + }, + 1, + ] as const; - this.logger.warn(`Page ${sn}: Browsing of ${url} failed`, { err }); - return new AssertionFailureError({ - message: `Failed to goto ${url}: ${err}`, - cause: err, - }); - }).then(async (stuff) => { - // This check is necessary because without snapshot, the condition of the page is unclear - // Calling evaluate directly may stall the process. - if (!snapshot) { - if (stuff instanceof Error) { - throw stuff; - } - } - await Promise.race([Promise.allSettled([...pageScriptEvaluations, ...frameScriptEvaluations]), delayPromise]) - .catch(() => void 0); - return stuff; - }); - if (options.waitForSelector) { - const t0 = Date.now(); - waitForPromise = nextSnapshotDeferred.promise.then(() => { - const t1 = Date.now(); - const elapsed = t1 - t0; - const remaining = timeout - elapsed; - const thisTimeout = remaining > 100 ? remaining : 100; - const p = (Array.isArray(options.waitForSelector) ? - Promise.all(options.waitForSelector.map((x) => page.waitForSelector(x, { timeout: thisTimeout }))) : - page.waitForSelector(options.waitForSelector!, { timeout: thisTimeout })) - .then(() => { - successfullyDone = true; - }) - .catch((err) => { - waitForPromise = undefined; - this.logger.warn(`Page ${sn}: Failed to wait for selector ${options.waitForSelector}`, { err }); - }); - return p as any; - }); - finalizationPromise = Promise.allSettled([waitForPromise, gotoPromise]).then(doFinalization); - } else { - finalizationPromise = gotoPromise.then(doFinalization); - } - - try { - let lastHTML = snapshot?.html; - while (true) { - const ckpt = [nextSnapshotDeferred.promise, waitForPromise ?? gotoPromise]; - - if (options.minIntervalMs) { - ckpt.push(delay(options.minIntervalMs)); - } - let error; - await Promise.race(ckpt).catch((err) => error = err); - if (successfullyDone && !error) { - if (!snapshot && !screenshot) { - throw new AssertionFailureError(`Could not extract any meaningful content from the page`); - } - yield { - ...snapshot, - status: navigationResponse?.status(), - statusText: navigationResponse?.statusText(), - pdfs: _.uniq(pdfUrls), screenshot, pageshot - } as PageSnapshot; - break; - } - if (options.favorScreenshot && snapshot?.title && snapshot?.html !== lastHTML) { - screenshot = (await this.takeScreenShot(page)) || screenshot; - pageshot = (await this.takeScreenShot(page, { fullPage: true })) || pageshot; - lastHTML = snapshot.html; - } - if (snapshot || screenshot) { - yield { - ...snapshot, - status: navigationResponse?.status(), - statusText: navigationResponse?.statusText(), - pdfs: _.uniq(pdfUrls), screenshot, pageshot, - isIntermediate: true, - } as PageSnapshot; - } - if (error) { - throw error; - } - if (successfullyDone) { - break; - } - } - await finalizationPromise; - yield { - ...snapshot, - status: navigationResponse?.status(), - statusText: navigationResponse?.statusText(), - pdfs: _.uniq(pdfUrls), screenshot, pageshot - } as PageSnapshot; - } finally { - this.pagePhase.set(page, 'background'); - Promise.allSettled([gotoPromise, waitForPromise, finalizationPromise]).finally(() => { - page.off('snapshot', hdl); - this.ditchPage(page); - }); - nextSnapshotDeferred.resolve(); - } - } - - protected async takeScreenShot(page: Page, opts?: Parameters[0]): Promise { - const r = await page.screenshot(opts).catch((err) => { - this.logger.warn(`Failed to take screenshot`, { err }); + return req.continue(continueArgs[0], continueArgs[1]); }); + } - if (r) { - return Buffer.from(r); + if (req.isInterceptResolutionHandled()) { + return; + } + const overrides = req.continueRequestOverrides(); + const continueArgs = [ + { + ...overrides, + headers: { + ...req.headers(), + ...overrides?.headers, + ...options.extraHeaders, + }, + }, + 1, + ] as const; + + return req.continue(continueArgs[0], continueArgs[1]); + }); + let pageScriptEvaluations: Promise[] = []; + let frameScriptEvaluations: Promise[] = []; + if (options.injectPageScripts?.length) { + page.on("framenavigated", (frame) => { + if (frame !== page.mainFrame()) { + return; } - return undefined; + pageScriptEvaluations.push( + Promise.allSettled( + options.injectPageScripts!.map((x) => + frame.evaluate(x).catch((err) => { + this.logger.warn(`Error in evaluation of page scripts`, { + err, + }); + }), + ), + ), + ); + }); + } + if (options.injectFrameScripts?.length) { + page.on("framenavigated", (frame) => { + frameScriptEvaluations.push( + Promise.allSettled( + options.injectFrameScripts!.map((x) => + frame.evaluate(x).catch((err) => { + this.logger.warn(`Error in evaluation of frame scripts`, { + err, + }); + }), + ), + ), + ); + }); + } + const sn = this.snMap.get(page); + this.logger.info(`Page ${sn}: Scraping ${url}`, { url }); + if (options.locale) { + // Add headers via request interception to walk around this bug + // https://github.com/puppeteer/puppeteer/issues/10235 + // await page.setExtraHTTPHeaders({ + // 'Accept-Language': options.locale + // }); + + await page.evaluateOnNewDocument(() => { + Object.defineProperty(navigator, "language", { + get: function () { + return options.locale; + }, + }); + Object.defineProperty(navigator, "languages", { + get: function () { + return [options.locale]; + }, + }); + }); } - async snapshotChildFrames(page: Page): Promise { - const childFrames = page.mainFrame().childFrames(); - const r = await Promise.all(childFrames.map(async (x) => { - const thisUrl = x.url(); - if (!thisUrl || thisUrl === 'about:blank') { - return undefined; - } - try { - await x.evaluate(SCRIPT_TO_INJECT_INTO_FRAME); + if (options.cookies) { + const mapped = options.cookies.map((x) => { + const draft: CookieParam = { + name: x.name, + value: encodeURIComponent(x.value), + secure: x.secure, + domain: x.domain, + path: x.path, + expires: x.expires + ? Math.floor(x.expires.valueOf() / 1000) + : undefined, + sameSite: x.sameSite as any, + }; + if (!draft.expires && x.maxAge) { + draft.expires = Math.floor(Date.now() / 1000) + x.maxAge; + } + if (!draft.domain) { + draft.url = parsedUrl.toString(); + } - return await x.evaluate(`giveSnapshot()`); - } catch (err) { - this.logger.warn(`Failed to snapshot child frame ${thisUrl}`, { err }); - return undefined; - } - })) as PageSnapshot[]; - - return r.filter(Boolean); + return draft; + }); + try { + await page.setCookie(...mapped); + } catch (err: any) { + this.logger.warn(`Page ${sn}: Failed to set cookies`, { err }); + throw new ParamValidationError({ + path: "cookies", + message: `Failed to set cookies: ${err?.message}`, + }); + } + } + if (options.overrideUserAgent) { + await page.setUserAgent(options.overrideUserAgent); + } + if (options.viewport) { + await page.setViewport(options.viewport); } + let nextSnapshotDeferred = Defer(); + const crippleListener = () => + nextSnapshotDeferred.reject( + new ServiceCrashedError({ message: `Browser crashed, try again` }), + ); + this.once("crippled", crippleListener); + nextSnapshotDeferred.promise.finally(() => { + this.off("crippled", crippleListener); + }); + let successfullyDone; + const hdl = (s: any) => { + if (snapshot === s) { + return; + } + snapshot = s; + if (snapshot) { + const kit = this.pageReqCtrl.get(page); + snapshot.lastContentResourceLoaded = kit?.lastContentResourceLoadedAt; + snapshot.lastMediaResourceLoaded = kit?.lastMediaResourceLoadedAt; + } + if (s?.maxElemDepth && s.maxElemDepth > 256) { + return; + } + if (s?.elemCount && s.elemCount > 10_000) { + return; + } + nextSnapshotDeferred.resolve(s); + nextSnapshotDeferred = Defer(); + this.once("crippled", crippleListener); + nextSnapshotDeferred.promise.finally(() => { + this.off("crippled", crippleListener); + }); + }; + page.on("snapshot", hdl); + page.once("abuse", (event: any) => { + this.emit("abuse", { ...event, url: parsedUrl }); + if (snapshot?.href && parsedUrl.href !== snapshot.href) { + this.emit("abuse", { ...event, url: snapshot.href }); + } + + nextSnapshotDeferred.reject( + new SecurityCompromiseError(`Abuse detected: ${event.reason}`), + ); + }); + + const timeout = options.timeoutMs || 30_000; + const goToOptions: GoToOptions = { + waitUntil: ["load", "domcontentloaded", "networkidle0"], + timeout, + }; + + if (options.referer) { + goToOptions.referer = options.referer; + } + + let waitForPromise: Promise | undefined; + let finalizationPromise: Promise | undefined; + const doFinalization = async () => { + if (waitForPromise) { + // SuccessfullyDone is meant for the finish of the page. + // It doesn't matter if you are expecting something and it didn't show up. + await waitForPromise.catch(() => void 0); + } + successfullyDone ??= true; + try { + const pSubFrameSnapshots = this.snapshotChildFrames(page); + snapshot = (await page.evaluate("giveSnapshot(true)")) as PageSnapshot; + screenshot = (await this.takeScreenShot(page)) || screenshot; + pageshot = + (await this.takeScreenShot(page, { fullPage: true })) || pageshot; + if (snapshot) { + snapshot.childFrames = await pSubFrameSnapshots; + } + } catch (err: any) { + this.logger.warn(`Page ${sn}: Failed to finalize ${url}`, { err }); + } + if (!snapshot?.html) { + return; + } + + this.logger.info(`Page ${sn}: Snapshot of ${url} done`, { + url, + title: snapshot?.title, + href: snapshot?.href, + }); + this.emit( + "crawled", + { + ...snapshot, + status: navigationResponse?.status(), + statusText: navigationResponse?.statusText(), + pdfs: _.uniq(pdfUrls), + screenshot, + pageshot, + }, + { ...options, url: parsedUrl }, + ); + }; + const delayPromise = delay(timeout); + const gotoPromise = page + .goto(url, goToOptions) + .catch((err) => { + if (err instanceof TimeoutError) { + this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err }); + return new AssertionFailureError({ + message: `Failed to goto ${url}: ${err}`, + cause: err, + }); + } + if (err?.message?.startsWith("net::ERR_ABORTED")) { + if (pdfUrls.length) { + // Not throw for pdf mode. + return; + } + } + + this.logger.warn(`Page ${sn}: Browsing of ${url} failed`, { err }); + return new AssertionFailureError({ + message: `Failed to goto ${url}: ${err}`, + cause: err, + }); + }) + .then(async (stuff) => { + // This check is necessary because without snapshot, the condition of the page is unclear + // Calling evaluate directly may stall the process. + if (!snapshot) { + if (stuff instanceof Error) { + throw stuff; + } + } + await Promise.race([ + Promise.allSettled([ + ...pageScriptEvaluations, + ...frameScriptEvaluations, + ]), + delayPromise, + ]).catch(() => void 0); + return stuff; + }); + if (options.waitForSelector) { + const t0 = Date.now(); + waitForPromise = nextSnapshotDeferred.promise.then(() => { + const t1 = Date.now(); + const elapsed = t1 - t0; + const remaining = timeout - elapsed; + const thisTimeout = remaining > 100 ? remaining : 100; + const p = ( + Array.isArray(options.waitForSelector) + ? Promise.all( + options.waitForSelector.map((x) => + page.waitForSelector(x, { timeout: thisTimeout }), + ), + ) + : page.waitForSelector(options.waitForSelector!, { + timeout: thisTimeout, + }) + ) + .then(() => { + successfullyDone = true; + }) + .catch((err) => { + waitForPromise = undefined; + this.logger.warn( + `Page ${sn}: Failed to wait for selector ${options.waitForSelector}`, + { err }, + ); + }); + return p as any; + }); + finalizationPromise = Promise.allSettled([ + waitForPromise, + gotoPromise, + ]).then(doFinalization); + } else { + finalizationPromise = gotoPromise.then(doFinalization); + } + + try { + let lastHTML = snapshot?.html; + while (true) { + const ckpt = [ + nextSnapshotDeferred.promise, + waitForPromise ?? gotoPromise, + ]; + + if (options.minIntervalMs) { + ckpt.push(delay(options.minIntervalMs)); + } + let error; + await Promise.race(ckpt).catch((err) => (error = err)); + if (successfullyDone && !error) { + if (!snapshot && !screenshot) { + throw new AssertionFailureError( + `Could not extract any meaningful content from the page`, + ); + } + yield { + ...snapshot, + status: navigationResponse?.status(), + statusText: navigationResponse?.statusText(), + pdfs: _.uniq(pdfUrls), + screenshot, + pageshot, + } as PageSnapshot; + break; + } + if ( + options.favorScreenshot && + snapshot?.title && + snapshot?.html !== lastHTML + ) { + screenshot = (await this.takeScreenShot(page)) || screenshot; + pageshot = + (await this.takeScreenShot(page, { fullPage: true })) || pageshot; + lastHTML = snapshot.html; + } + if (snapshot || screenshot) { + yield { + ...snapshot, + status: navigationResponse?.status(), + statusText: navigationResponse?.statusText(), + pdfs: _.uniq(pdfUrls), + screenshot, + pageshot, + isIntermediate: true, + } as PageSnapshot; + } + if (error) { + throw error; + } + if (successfullyDone) { + break; + } + } + await finalizationPromise; + yield { + ...snapshot, + status: navigationResponse?.status(), + statusText: navigationResponse?.statusText(), + pdfs: _.uniq(pdfUrls), + screenshot, + pageshot, + } as PageSnapshot; + } finally { + this.pagePhase.set(page, "background"); + Promise.allSettled([ + gotoPromise, + waitForPromise, + finalizationPromise, + ]).finally(() => { + page.off("snapshot", hdl); + this.ditchPage(page); + }); + nextSnapshotDeferred.resolve(); + } + } + + protected async takeScreenShot( + page: Page, + opts?: Parameters[0], + ): Promise { + const r = await page.screenshot(opts).catch((err) => { + this.logger.warn(`Failed to take screenshot`, { err }); + }); + + if (r) { + return Buffer.from(r); + } + + return undefined; + } + + async snapshotChildFrames(page: Page): Promise { + const childFrames = page.mainFrame().childFrames(); + const r = (await Promise.all( + childFrames.map(async (x) => { + const thisUrl = x.url(); + if (!thisUrl || thisUrl === "about:blank") { + return undefined; + } + try { + await x.evaluate(SCRIPT_TO_INJECT_INTO_FRAME); + + return await x.evaluate(`giveSnapshot()`); + } catch (err) { + this.logger.warn(`Failed to snapshot child frame ${thisUrl}`, { + err, + }); + return undefined; + } + }), + )) as PageSnapshot[]; + + return r.filter(Boolean); + } } const puppeteerControl = container.resolve(PuppeteerControl); diff --git a/src/services/registry.ts b/src/services/registry.ts index 0ab462e..2e90ed1 100644 --- a/src/services/registry.ts +++ b/src/services/registry.ts @@ -1,60 +1,59 @@ -import { propertyInjectorFactory } from 'civkit/property-injector'; -import { KoaRPCRegistry } from 'civkit/civ-rpc/koa'; -import { container, singleton } from 'tsyringe'; -import { IntegrityEnvelope } from 'civkit/civ-rpc'; -import bodyParser from '@koa/bodyparser'; +import { propertyInjectorFactory } from "civkit/property-injector"; +import { KoaRPCRegistry } from "civkit/civ-rpc/koa"; +import { container, singleton } from "tsyringe"; +import { IntegrityEnvelope } from "civkit/civ-rpc"; +import bodyParser from "@koa/bodyparser"; -import { GlobalLogger } from './logger'; -import { TempFileManager } from './temp-file'; -import { AsyncLocalContext } from './async-context'; -import { BlackHoleDetector } from './blackhole-detector'; -export { Context } from 'koa'; +import { GlobalLogger } from "./logger"; +import { TempFileManager } from "./temp-file"; +import { AsyncLocalContext } from "./async-context"; +import { BlackHoleDetector } from "./blackhole-detector"; +export { Context } from "koa"; export const InjectProperty = propertyInjectorFactory(container); @singleton() export class RPCRegistry extends KoaRPCRegistry { + title = "Xread API"; + container = container; + logger = this.globalLogger.child({ service: this.constructor.name }); + static override envelope = IntegrityEnvelope; + override _BODY_PARSER_LIMIT = "102mb"; + override _RESPONSE_STREAM_MODE = "koa" as const; - title = 'Xread API'; - container = container; - logger = this.globalLogger.child({ service: this.constructor.name }); - static override envelope = IntegrityEnvelope; - override _BODY_PARSER_LIMIT = '102mb'; - override _RESPONSE_STREAM_MODE = 'koa' as const; + override koaMiddlewares = [ + this.__CORSAllowAllMiddleware.bind(this), + bodyParser({ + encoding: "utf-8", + enableTypes: ["json", "form"], + jsonLimit: this._BODY_PARSER_LIMIT, + xmlLimit: this._BODY_PARSER_LIMIT, + formLimit: this._BODY_PARSER_LIMIT, + }), + this.__multiParse.bind(this), + this.__binaryParse.bind(this), + ]; - override koaMiddlewares = [ - this.__CORSAllowAllMiddleware.bind(this), - bodyParser({ - encoding: 'utf-8', - enableTypes: ['json', 'form'], - jsonLimit: this._BODY_PARSER_LIMIT, - xmlLimit: this._BODY_PARSER_LIMIT, - formLimit: this._BODY_PARSER_LIMIT, - }), - this.__multiParse.bind(this), - this.__binaryParse.bind(this), - ]; + constructor( + protected globalLogger: GlobalLogger, + protected ctxMgr: AsyncLocalContext, + protected tempFileManager: TempFileManager, + protected blackHoleDetector: BlackHoleDetector, + ) { + super(...arguments); - constructor( - protected globalLogger: GlobalLogger, - protected ctxMgr: AsyncLocalContext, - protected tempFileManager: TempFileManager, - protected blackHoleDetector: BlackHoleDetector, - ) { - super(...arguments); - - this.on('run', () => this.blackHoleDetector.incomingRequest()); - this.on('ran', () => this.blackHoleDetector.doneWithRequest()); - this.on('fail', () => this.blackHoleDetector.doneWithRequest()); - } - - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } + this.on("run", () => this.blackHoleDetector.incomingRequest()); + this.on("ran", () => this.blackHoleDetector.doneWithRequest()); + this.on("fail", () => this.blackHoleDetector.doneWithRequest()); + } + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } } const instance = container.resolve(RPCRegistry); export default instance; -export const { Method, RPCMethod, RPCReflect, Param, Ctx, } = instance.decorators(); +export const { Method, RPCMethod, RPCReflect, Param, Ctx } = + instance.decorators(); diff --git a/src/services/robots-text.ts b/src/services/robots-text.ts index 365af3c..3ed3543 100644 --- a/src/services/robots-text.ts +++ b/src/services/robots-text.ts @@ -1,127 +1,144 @@ -import { singleton } from 'tsyringe'; -import { URL } from 'url'; -import { AssertionFailureError, DownstreamServiceFailureError, ResourcePolicyDenyError } from 'civkit/civ-rpc'; -import { AsyncService } from 'civkit/async-service'; -import { HashManager } from 'civkit/hash'; -import { marshalErrorLike } from 'civkit/lang'; +import { singleton } from "tsyringe"; +import { URL } from "url"; +import { + AssertionFailureError, + DownstreamServiceFailureError, + ResourcePolicyDenyError, +} from "civkit/civ-rpc"; +import { AsyncService } from "civkit/async-service"; +import { HashManager } from "civkit/hash"; +import { marshalErrorLike } from "civkit/lang"; -import { GlobalLogger } from './logger'; -import { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket'; -import { Threaded } from '../services/threaded'; +import { GlobalLogger } from "./logger"; +import { FirebaseStorageBucketControl } from "../shared/services/firebase-storage-bucket"; +import { Threaded } from "../services/threaded"; - -export const md5Hasher = new HashManager('md5', 'hex'); +export const md5Hasher = new HashManager("md5", "hex"); @singleton() export class RobotsTxtService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + constructor( + protected globalLogger: GlobalLogger, + protected firebaseStorageBucketControl: FirebaseStorageBucketControl, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected firebaseStorageBucketControl: FirebaseStorageBucketControl, - ) { - super(...arguments); + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } + + async getCachedRobotTxt(origin: string) { + const digest = md5Hasher.hash(origin.toLowerCase()); + const cacheLoc = `robots-txt/${digest}`; + let buff; + buff = await this.firebaseStorageBucketControl + .downloadFile(cacheLoc) + .catch(() => undefined); + if (buff) { + return buff.toString(); } - override async init() { - await this.dependencyReady(); - this.emit('ready'); + const r = await fetch(new URL("robots.txt", origin).href, { + signal: AbortSignal.timeout(5000), + }); + if (!r.ok) { + throw new DownstreamServiceFailureError( + `Failed to fetch robots.txt from ${origin}: ${r.status} ${r.statusText}`, + ); } + buff = Buffer.from(await r.arrayBuffer()); - async getCachedRobotTxt(origin: string) { - const digest = md5Hasher.hash(origin.toLowerCase()); - const cacheLoc = `robots-txt/${digest}`; - let buff; - buff = await this.firebaseStorageBucketControl.downloadFile(cacheLoc).catch(() => undefined); - if (buff) { - return buff.toString(); - } - - const r = await fetch(new URL('robots.txt', origin).href, { signal: AbortSignal.timeout(5000) }); - if (!r.ok) { - throw new DownstreamServiceFailureError(`Failed to fetch robots.txt from ${origin}: ${r.status} ${r.statusText}`); - } - buff = Buffer.from(await r.arrayBuffer()); - - this.firebaseStorageBucketControl.saveFile(cacheLoc, buff, { - contentType: 'text/plain' - }).catch((err) => { - this.logger.warn(`Failed to save robots.txt to cache: ${err}`, { err: marshalErrorLike(err) }); + this.firebaseStorageBucketControl + .saveFile(cacheLoc, buff, { + contentType: "text/plain", + }) + .catch((err) => { + this.logger.warn(`Failed to save robots.txt to cache: ${err}`, { + err: marshalErrorLike(err), }); + }); - return buff.toString(); - } - - @Threaded() - async assertAccessAllowed(url: URL, inputMyUa = '*') { - let robotTxt: string = ''; - try { - robotTxt = await this.getCachedRobotTxt(url.origin); - } catch (err) { - if (err instanceof DownstreamServiceFailureError) { - // Remote server is reachable but cannot provide a robot.txt; this is treated as public access - return true; - } - throw new AssertionFailureError(`Failed to load robots.txt from ${url.origin}: ${err}`); - } - const myUa = inputMyUa.toLowerCase(); - const lines = robotTxt.split(/\r?\n/g); - - let currentUa = myUa || '*'; - let uaLine = 'User-Agent: *'; - const pathNormalized = `${url.pathname}?`; - - for (const line of lines) { - const trimmed = line.trim(); - if (trimmed.startsWith('#') || !trimmed) { - continue; - } - const [k, ...rest] = trimmed.split(':'); - const key = k.trim().toLowerCase(); - const value = rest.join(':').trim(); - - if (key === 'user-agent') { - currentUa = value.toLowerCase(); - if (value === '*') { - currentUa = myUa; - } - uaLine = line; - continue; - } - - if (currentUa !== myUa) { - continue; - } - - if (key === 'disallow') { - if (!value) { - return true; - } - if (value.includes('*')) { - const [head, tail] = value.split('*'); - if (url.pathname.startsWith(head) && url.pathname.endsWith(tail)) { - throw new ResourcePolicyDenyError(`Access to ${url.href} is disallowed by site robots.txt: For ${uaLine}, ${line}`); - } - } else if (pathNormalized.startsWith(value)) { - throw new ResourcePolicyDenyError(`Access to ${url.href} is disallowed by site robots.txt: For ${uaLine}, ${line}`); - } - - continue; - } - - if (key === 'allow') { - if (!value) { - return true; - } - if (pathNormalized.startsWith(value)) { - return true; - } - continue; - } - } + return buff.toString(); + } + @Threaded() + async assertAccessAllowed(url: URL, inputMyUa = "*") { + let robotTxt: string = ""; + try { + robotTxt = await this.getCachedRobotTxt(url.origin); + } catch (err) { + if (err instanceof DownstreamServiceFailureError) { + // Remote server is reachable but cannot provide a robot.txt; this is treated as public access return true; + } + throw new AssertionFailureError( + `Failed to load robots.txt from ${url.origin}: ${err}`, + ); + } + const myUa = inputMyUa.toLowerCase(); + const lines = robotTxt.split(/\r?\n/g); + + let currentUa = myUa || "*"; + let uaLine = "User-Agent: *"; + const pathNormalized = `${url.pathname}?`; + + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.startsWith("#") || !trimmed) { + continue; + } + const [k, ...rest] = trimmed.split(":"); + const key = k.trim().toLowerCase(); + const value = rest.join(":").trim(); + + if (key === "user-agent") { + currentUa = value.toLowerCase(); + if (value === "*") { + currentUa = myUa; + } + uaLine = line; + continue; + } + + if (currentUa !== myUa) { + continue; + } + + if (key === "disallow") { + if (!value) { + return true; + } + if (value.includes("*")) { + const [head, tail] = value.split("*"); + if (url.pathname.startsWith(head) && url.pathname.endsWith(tail)) { + throw new ResourcePolicyDenyError( + `Access to ${url.href} is disallowed by site robots.txt: For ${uaLine}, ${line}`, + ); + } + } else if (pathNormalized.startsWith(value)) { + throw new ResourcePolicyDenyError( + `Access to ${url.href} is disallowed by site robots.txt: For ${uaLine}, ${line}`, + ); + } + + continue; + } + + if (key === "allow") { + if (!value) { + return true; + } + if (pathNormalized.startsWith(value)) { + return true; + } + continue; + } } + return true; + } } diff --git a/src/services/serp/compat.ts b/src/services/serp/compat.ts index 1543c34..c2bbf21 100644 --- a/src/services/serp/compat.ts +++ b/src/services/serp/compat.ts @@ -1,12 +1,14 @@ export interface WebSearchEntry { + link: string; + title: string; + source?: string; + date?: string; + snippet?: string; + imageUrl?: string; + siteLinks?: { link: string; title: string; - source?: string; - date?: string; snippet?: string; - imageUrl?: string; - siteLinks?: { - link: string; title: string; snippet?: string; - }[]; - variant?: 'web' | 'images' | 'news'; -} \ No newline at end of file + }[]; + variant?: "web" | "images" | "news"; +} diff --git a/src/services/serp/google.ts b/src/services/serp/google.ts index 81a2c30..775e804 100644 --- a/src/services/serp/google.ts +++ b/src/services/serp/google.ts @@ -1,327 +1,389 @@ -import { singleton } from 'tsyringe'; -import { AsyncService } from 'civkit/async-service'; -import { GlobalLogger } from '../logger'; -import { JSDomControl } from '../jsdom'; -import { isMainThread } from 'worker_threads'; -import _ from 'lodash'; -import { WebSearchEntry } from './compat'; -import { ScrappingOptions, SERPSpecializedPuppeteerControl } from './puppeteer'; -import { CurlControl } from '../curl'; -import { readFile } from 'fs/promises'; -import { ApplicationError } from 'civkit/civ-rpc'; -import { ServiceBadApproachError, ServiceBadAttemptError } from '../errors'; -import { parseJSONText } from 'civkit/vectorize'; -import { retryWith } from 'civkit/decorators'; -import { ProxyProviderService } from '../../shared/services/proxy-provider'; -import { SERP_BOOT_TIMEOUT_MS } from '../boot-timeouts'; +import { singleton } from "tsyringe"; +import { AsyncService } from "civkit/async-service"; +import { GlobalLogger } from "../logger"; +import { JSDomControl } from "../jsdom"; +import { isMainThread } from "worker_threads"; +import _ from "lodash"; +import { WebSearchEntry } from "./compat"; +import { ScrappingOptions, SERPSpecializedPuppeteerControl } from "./puppeteer"; +import { CurlControl } from "../curl"; +import { readFile } from "fs/promises"; +import { ApplicationError } from "civkit/civ-rpc"; +import { ServiceBadApproachError, ServiceBadAttemptError } from "../errors"; +import { parseJSONText } from "civkit/vectorize"; +import { retryWith } from "civkit/decorators"; +import { ProxyProviderService } from "../../shared/services/proxy-provider"; +import { SERP_BOOT_TIMEOUT_MS } from "../boot-timeouts"; @singleton() export class GoogleSERP extends AsyncService { - logger = this.globalLogger.child({ service: this.constructor.name }); - googleDomain = process.env.OVERRIDE_GOOGLE_DOMAIN || 'www.google.com'; + logger = this.globalLogger.child({ service: this.constructor.name }); + googleDomain = process.env.OVERRIDE_GOOGLE_DOMAIN || "www.google.com"; - constructor( - protected globalLogger: GlobalLogger, - protected puppeteerControl: SERPSpecializedPuppeteerControl, - protected jsDomControl: JSDomControl, - protected curlControl: CurlControl, - protected proxyProvider: ProxyProviderService, - ) { - const filteredDeps = isMainThread ? arguments : _.without(arguments, puppeteerControl); - super(...filteredDeps); + constructor( + protected globalLogger: GlobalLogger, + protected puppeteerControl: SERPSpecializedPuppeteerControl, + protected jsDomControl: JSDomControl, + protected curlControl: CurlControl, + protected proxyProvider: ProxyProviderService, + ) { + const filteredDeps = isMainThread + ? arguments + : _.without(arguments, puppeteerControl); + super(...filteredDeps); + } + + override async init() { + await this.dependencyReady(SERP_BOOT_TIMEOUT_MS); + + this.emit("ready"); + } + + override dependencyReady(timeoutMilliseconds = SERP_BOOT_TIMEOUT_MS) { + return super.dependencyReady(timeoutMilliseconds); + } + + @retryWith((err) => { + if (err instanceof ServiceBadApproachError) { + return false; + } + if (err instanceof ServiceBadAttemptError) { + // Keep trying + return true; + } + if (err instanceof ApplicationError) { + // Quit with this error + return false; + } + return undefined; + }, 3) + async sideLoadWithAllocatedProxy(url: URL, opts?: ScrappingOptions) { + if (opts?.allocProxy === "none") { + return this.curlControl.sideLoad(url, opts); } - override async init() { - await this.dependencyReady(SERP_BOOT_TIMEOUT_MS); - - this.emit('ready'); + if (!this.proxyProvider.isConfigured()) { + this.logger.info( + `No proxy configured, falling back to direct Google request`, + { url: url.href }, + ); + return this.curlControl.sideLoad(url, opts); } - override dependencyReady(timeoutMilliseconds = SERP_BOOT_TIMEOUT_MS) { - return super.dependencyReady(timeoutMilliseconds); + const proxy = await this.proxyProvider.alloc( + process.env.PREFERRED_PROXY_COUNTRY || "auto", + ); + this.logger.debug(`Proxy allocated`, { proxy: proxy.href }); + const r = await this.curlControl.sideLoad(url, { + ...opts, + proxyUrl: proxy.href, + }); + + if (r.status === 429) { + throw new ServiceBadAttemptError( + "Google returned a 429 error. This may happen due to various reasons, including rate limiting or other issues.", + ); } - @retryWith((err) => { - if (err instanceof ServiceBadApproachError) { - return false; - } - if (err instanceof ServiceBadAttemptError) { - // Keep trying - return true; - } - if (err instanceof ApplicationError) { - // Quit with this error - return false; - } - return undefined; - }, 3) - async sideLoadWithAllocatedProxy(url: URL, opts?: ScrappingOptions) { - if (opts?.allocProxy === 'none') { - return this.curlControl.sideLoad(url, opts); - } - - if (!this.proxyProvider.isConfigured()) { - this.logger.info(`No proxy configured, falling back to direct Google request`, { url: url.href }); - return this.curlControl.sideLoad(url, opts); - } - - const proxy = await this.proxyProvider.alloc( - process.env.PREFERRED_PROXY_COUNTRY || 'auto' - ); - this.logger.debug(`Proxy allocated`, { proxy: proxy.href }); - const r = await this.curlControl.sideLoad(url, { - ...opts, - proxyUrl: proxy.href, - }); - - if (r.status === 429) { - throw new ServiceBadAttemptError('Google returned a 429 error. This may happen due to various reasons, including rate limiting or other issues.'); - } - - if (opts && opts.allocProxy) { - opts.proxyUrl ??= proxy.href; - } - - return { ...r, proxy }; + if (opts && opts.allocProxy) { + opts.proxyUrl ??= proxy.href; } - digestQuery(query: { [k: string]: any; }) { - const url = new URL(`https://${this.googleDomain}/search`); - const clone = { ...query }; - const num = clone.num || 10; - if (clone.page) { - const page = parseInt(clone.page); - delete clone.page; - clone.start = (page - 1) * num; - if (clone.start === 0) { - delete clone.start; - } - } - if (clone.location) { - delete clone.location; - } + return { ...r, proxy }; + } - for (const [k, v] of Object.entries(clone)) { - if (v === undefined || v === null) { - continue; - } - url.searchParams.set(k, `${v}`); - } - - return url; + digestQuery(query: { [k: string]: any }) { + const url = new URL(`https://${this.googleDomain}/search`); + const clone = { ...query }; + const num = clone.num || 10; + if (clone.page) { + const page = parseInt(clone.page); + delete clone.page; + clone.start = (page - 1) * num; + if (clone.start === 0) { + delete clone.start; + } + } + if (clone.location) { + delete clone.location; } - async webSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) { - const url = this.digestQuery(query); - - const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts); - if (opts && sideLoaded.sideLoadOpts) { - opts.sideLoad = sideLoaded.sideLoadOpts; - } - - const snapshot = await this.puppeteerControl.controlledScrap(url, getWebSearchResults, opts); - - return snapshot; + for (const [k, v] of Object.entries(clone)) { + if (v === undefined || v === null) { + continue; + } + url.searchParams.set(k, `${v}`); } - async newsSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) { - const url = this.digestQuery(query); + return url; + } - url.searchParams.set('tbm', 'nws'); + async webSearch(query: { [k: string]: any }, opts?: ScrappingOptions) { + const url = this.digestQuery(query); - const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts); - if (opts && sideLoaded.sideLoadOpts) { - opts.sideLoad = sideLoaded.sideLoadOpts; - } - - const snapshot = await this.puppeteerControl.controlledScrap(url, getNewsSearchResults, opts); - - return snapshot; + const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts); + if (opts && sideLoaded.sideLoadOpts) { + opts.sideLoad = sideLoaded.sideLoadOpts; } - async imageSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) { - const url = this.digestQuery(query); + const snapshot = await this.puppeteerControl.controlledScrap( + url, + getWebSearchResults, + opts, + ); - url.searchParams.set('tbm', 'isch'); - url.searchParams.set('asearch', 'isch'); - url.searchParams.set('async', `_fmt:json,p:1,ijn:${query.start ? Math.floor(query.start / (query.num || 10)) : 0}`); + return snapshot; + } - const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts); + async newsSearch(query: { [k: string]: any }, opts?: ScrappingOptions) { + const url = this.digestQuery(query); - if (sideLoaded.status !== 200 || !sideLoaded.file) { - throw new ServiceBadAttemptError('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.'); - } + url.searchParams.set("tbm", "nws"); - const jsonTxt = (await readFile((await sideLoaded.file.filePath))).toString(); - const rJSON = parseJSONText(jsonTxt.slice(jsonTxt.indexOf('{"ischj":'))); - - return _.get(rJSON, 'ischj.metadata').map((x: any) => { - - return { - link: _.get(x, 'result.referrer_url'), - title: _.get(x, 'result.page_title'), - snippet: _.get(x, 'text_in_grid.snippet'), - source: _.get(x, 'result.site_title'), - imageWidth: _.get(x, 'original_image.width'), - imageHeight: _.get(x, 'original_image.height'), - imageUrl: _.get(x, 'original_image.url'), - variant: 'images', - }; - }) as WebSearchEntry[]; + const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts); + if (opts && sideLoaded.sideLoadOpts) { + opts.sideLoad = sideLoaded.sideLoadOpts; } + + const snapshot = await this.puppeteerControl.controlledScrap( + url, + getNewsSearchResults, + opts, + ); + + return snapshot; + } + + async imageSearch(query: { [k: string]: any }, opts?: ScrappingOptions) { + const url = this.digestQuery(query); + + url.searchParams.set("tbm", "isch"); + url.searchParams.set("asearch", "isch"); + url.searchParams.set( + "async", + `_fmt:json,p:1,ijn:${query.start ? Math.floor(query.start / (query.num || 10)) : 0}`, + ); + + const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts); + + if (sideLoaded.status !== 200 || !sideLoaded.file) { + throw new ServiceBadAttemptError( + "Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.", + ); + } + + const jsonTxt = (await readFile(await sideLoaded.file.filePath)).toString(); + const rJSON = parseJSONText(jsonTxt.slice(jsonTxt.indexOf('{"ischj":'))); + + return _.get(rJSON, "ischj.metadata").map((x: any) => { + return { + link: _.get(x, "result.referrer_url"), + title: _.get(x, "result.page_title"), + snippet: _.get(x, "text_in_grid.snippet"), + source: _.get(x, "result.site_title"), + imageWidth: _.get(x, "original_image.width"), + imageHeight: _.get(x, "original_image.height"), + imageUrl: _.get(x, "original_image.url"), + variant: "images", + }; + }) as WebSearchEntry[]; + } } async function getWebSearchResults() { - if (location.pathname.startsWith('/sorry') || location.pathname.startsWith('/error')) { - throw new Error('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.'); - } + if ( + location.pathname.startsWith("/sorry") || + location.pathname.startsWith("/error") + ) { + throw new Error( + "Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.", + ); + } - // @ts-ignore - await Promise.race([window.waitForSelector('div[data-async-context^="query"]'), window.waitForSelector('#botstuff .mnr-c')]); + // @ts-ignore + await Promise.race([ + window.waitForSelector('div[data-async-context^="query"]'), + window.waitForSelector("#botstuff .mnr-c"), + ]); - const wrapper1 = document.querySelector('div[data-async-context^="query"]'); + const wrapper1 = document.querySelector('div[data-async-context^="query"]'); - if (!wrapper1) { + if (!wrapper1) { + return undefined; + } + + const query = decodeURIComponent( + wrapper1.getAttribute("data-async-context")?.split("query:")[1] || "", + ); + + if (!query) { + return undefined; + } + + const candidates = Array.from( + wrapper1.querySelectorAll("div[lang],div[data-surl]"), + ); + + return candidates + .map((x) => { + const primaryLink = x.querySelector('a:not([href="#"])'); + if (!primaryLink) { return undefined; - } + } + const url = primaryLink.getAttribute("href"); - const query = decodeURIComponent(wrapper1.getAttribute('data-async-context')?.split('query:')[1] || ''); + if (primaryLink.querySelector('div[role="heading"]')) { + // const spans = primaryLink.querySelectorAll('span'); + // const title = spans[0]?.textContent; + // const source = spans[1]?.textContent; + // const date = spans[spans.length - 1].textContent; - if (!query) { + // return { + // link: url, + // title, + // source, + // date, + // variant: 'video' + // }; return undefined; - } + } - const candidates = Array.from(wrapper1.querySelectorAll('div[lang],div[data-surl]')); - - return candidates.map((x) => { - const primaryLink = x.querySelector('a:not([href="#"])'); - if (!primaryLink) { + const title = primaryLink.querySelector("h3")?.textContent; + const source = Array.from(primaryLink.querySelectorAll("span")).find( + (x) => x.textContent, + )?.textContent; + const cite = primaryLink.querySelector("cite[role=text]")?.textContent; + let date = cite?.split("·")[1]?.trim(); + const snippets = Array.from( + x.querySelectorAll('div[data-sncf*="1"] span'), + ); + let snippet: string | null | undefined = + snippets[snippets.length - 1]?.textContent; + if (!snippet) { + snippet = x.querySelector("div.IsZvec")?.textContent?.trim() || null; + } + date ??= snippets[snippets.length - 2]?.textContent?.trim(); + const imageUrl = x + .querySelector('div[data-sncf*="1"] img[src]:not(img[src^="data"])') + ?.getAttribute("src"); + let siteLinks = Array.from( + x.querySelectorAll('div[data-sncf*="3"] a[href]'), + ) + .map((l) => { + const link = l.getAttribute("href"); + if (!link) { return undefined; - } - const url = primaryLink.getAttribute('href'); + } - if (primaryLink.querySelector('div[role="heading"]')) { - // const spans = primaryLink.querySelectorAll('span'); - // const title = spans[0]?.textContent; - // const source = spans[1]?.textContent; - // const date = spans[spans.length - 1].textContent; - - // return { - // link: url, - // title, - // source, - // date, - // variant: 'video' - // }; - return undefined; - } - - const title = primaryLink.querySelector('h3')?.textContent; - const source = Array.from(primaryLink.querySelectorAll('span')).find((x) => x.textContent)?.textContent; - const cite = primaryLink.querySelector('cite[role=text]')?.textContent; - let date = cite?.split('·')[1]?.trim(); - const snippets = Array.from(x.querySelectorAll('div[data-sncf*="1"] span')); - let snippet: string | null | undefined = snippets[snippets.length - 1]?.textContent; - if (!snippet) { - snippet = x.querySelector('div.IsZvec')?.textContent?.trim() || null; - } - date ??= snippets[snippets.length - 2]?.textContent?.trim(); - const imageUrl = x.querySelector('div[data-sncf*="1"] img[src]:not(img[src^="data"])')?.getAttribute('src'); - let siteLinks = Array.from(x.querySelectorAll('div[data-sncf*="3"] a[href]')).map((l) => { - const link = l.getAttribute('href'); - if (!link) { + return { + link, + title: l.textContent, + }; + }) + .filter(Boolean) as any; + const perhapsParent = x.parentElement?.closest("div[data-hveid]"); + if (!siteLinks?.length && perhapsParent) { + const candidates = Array.from(perhapsParent.querySelectorAll("td h3")); + if (candidates.length) { + siteLinks = candidates + .map((l) => { + const link = l.querySelector("a"); + if (!link) { return undefined; - } - - return { - link, - title: l.textContent, - }; - }).filter(Boolean) as any; - const perhapsParent = x.parentElement?.closest('div[data-hveid]'); - if (!siteLinks?.length && perhapsParent) { - const candidates = Array.from(perhapsParent.querySelectorAll('td h3')); - if (candidates.length) { - siteLinks = candidates.map((l) => { - const link = l.querySelector('a'); - if (!link) { - return undefined; - } - const href = link.getAttribute('href'); - if (!href) { - return undefined; - } - const snippet = l.nextElementSibling?.textContent; - return { - link: href, - title: link.textContent, - snippet, - }; - }).filter(Boolean) as any; - } + } + const href = link.getAttribute("href"); + if (!href) { + return undefined; + } + const snippet = l.nextElementSibling?.textContent; + return { + link: href, + title: link.textContent, + snippet, + }; + }) + .filter(Boolean) as any; } + } - return { - link: url, - title, - source, - date, - snippet: snippet ?? undefined, - imageUrl: imageUrl?.startsWith('data:') ? undefined : imageUrl, - siteLinks: siteLinks.length ? siteLinks : undefined, - variant: 'web', - }; - }).filter(Boolean) as WebSearchEntry[]; + return { + link: url, + title, + source, + date, + snippet: snippet ?? undefined, + imageUrl: imageUrl?.startsWith("data:") ? undefined : imageUrl, + siteLinks: siteLinks.length ? siteLinks : undefined, + variant: "web", + }; + }) + .filter(Boolean) as WebSearchEntry[]; } async function getNewsSearchResults() { - if (location.pathname.startsWith('/sorry') || location.pathname.startsWith('/error')) { - throw new Error('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.'); - } + if ( + location.pathname.startsWith("/sorry") || + location.pathname.startsWith("/error") + ) { + throw new Error( + "Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.", + ); + } - // @ts-ignore - await Promise.race([window.waitForSelector('div[data-async-context^="query"]'), window.waitForSelector('#botstuff .mnr-c')]); + // @ts-ignore + await Promise.race([ + window.waitForSelector('div[data-async-context^="query"]'), + window.waitForSelector("#botstuff .mnr-c"), + ]); - const wrapper1 = document.querySelector('div[data-async-context^="query"]'); + const wrapper1 = document.querySelector('div[data-async-context^="query"]'); - if (!wrapper1) { + if (!wrapper1) { + return undefined; + } + + const query = decodeURIComponent( + wrapper1.getAttribute("data-async-context")?.split("query:")[1] || "", + ); + + if (!query) { + return undefined; + } + + const candidates = Array.from( + wrapper1.querySelectorAll("div[data-news-doc-id]"), + ); + + return candidates + .map((x) => { + const primaryLink = x.querySelector('a:not([href="#"])'); + if (!primaryLink) { return undefined; - } + } + const url = primaryLink.getAttribute("href"); + const titleElem = primaryLink.querySelector('div[role="heading"]'); - const query = decodeURIComponent(wrapper1.getAttribute('data-async-context')?.split('query:')[1] || ''); - - if (!query) { + if (!titleElem) { return undefined; - } + } - const candidates = Array.from(wrapper1.querySelectorAll('div[data-news-doc-id]')); + const title = titleElem.textContent?.trim(); + const source = titleElem.previousElementSibling?.textContent?.trim(); + const snippet = titleElem.nextElementSibling?.textContent?.trim(); - return candidates.map((x) => { - const primaryLink = x.querySelector('a:not([href="#"])'); - if (!primaryLink) { - return undefined; - } - const url = primaryLink.getAttribute('href'); - const titleElem = primaryLink.querySelector('div[role="heading"]'); + const innerSpans = Array.from( + titleElem.parentElement?.querySelectorAll("span") || [], + ); + const date = innerSpans[innerSpans.length - 1]?.textContent?.trim(); - if (!titleElem) { - return undefined; - } - - const title = titleElem.textContent?.trim(); - const source = titleElem.previousElementSibling?.textContent?.trim(); - const snippet = titleElem.nextElementSibling?.textContent?.trim(); - - const innerSpans = Array.from(titleElem.parentElement?.querySelectorAll('span') || []); - const date = innerSpans[innerSpans.length - 1]?.textContent?.trim(); - - return { - link: url, - title, - source, - date, - snippet, - variant: 'news', - }; - }).filter(Boolean) as WebSearchEntry[]; + return { + link: url, + title, + source, + date, + snippet, + variant: "news", + }; + }) + .filter(Boolean) as WebSearchEntry[]; } diff --git a/src/services/serp/internal-search.ts b/src/services/serp/internal-search.ts index 7eb43b5..36cf1fa 100644 --- a/src/services/serp/internal-search.ts +++ b/src/services/serp/internal-search.ts @@ -1,43 +1,45 @@ -import { singleton } from 'tsyringe'; -import { GlobalLogger } from '../logger'; -import { AsyncService } from 'civkit/async-service'; -import { SerperSearchQueryParams } from '../../shared/3rd-party/serper-search'; -import { ServiceDisabledError } from '../errors'; -import { WebSearchEntry } from './compat'; +import { singleton } from "tsyringe"; +import { GlobalLogger } from "../logger"; +import { AsyncService } from "civkit/async-service"; +import { SerperSearchQueryParams } from "../../shared/3rd-party/serper-search"; +import { ServiceDisabledError } from "../errors"; +import { WebSearchEntry } from "./compat"; @singleton() export class InternalSearchService extends AsyncService { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - constructor( - protected globalLogger: GlobalLogger, - ) { - super(...arguments); - } + constructor(protected globalLogger: GlobalLogger) { + super(...arguments); + } - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } - protected disabled(): never { - throw new ServiceDisabledError('Internal search provider is not available in this standalone build. Use a public provider such as google, bing, or serper.'); - } + protected disabled(): never { + throw new ServiceDisabledError( + "Internal search provider is not available in this standalone build. Use a public provider such as google, bing, or serper.", + ); + } - async doSearch(_variant: 'web' | 'images' | 'news', _query: SerperSearchQueryParams) { - this.disabled(); - } + async doSearch( + _variant: "web" | "images" | "news", + _query: SerperSearchQueryParams, + ) { + this.disabled(); + } - async webSearch(_query: SerperSearchQueryParams) { - return this.disabled() as WebSearchEntry[]; - } + async webSearch(_query: SerperSearchQueryParams) { + return this.disabled() as WebSearchEntry[]; + } - async imageSearch(_query: SerperSearchQueryParams) { - return this.disabled() as WebSearchEntry[]; - } + async imageSearch(_query: SerperSearchQueryParams) { + return this.disabled() as WebSearchEntry[]; + } - async newsSearch(_query: SerperSearchQueryParams) { - return this.disabled() as WebSearchEntry[]; - } + async newsSearch(_query: SerperSearchQueryParams) { + return this.disabled() as WebSearchEntry[]; + } } - diff --git a/src/services/serp/puppeteer.ts b/src/services/serp/puppeteer.ts index 2616339..5c9b5f5 100644 --- a/src/services/serp/puppeteer.ts +++ b/src/services/serp/puppeteer.ts @@ -1,64 +1,75 @@ -import _ from 'lodash'; -import fs from 'fs'; -import { readFile } from 'fs/promises'; -import { container, singleton } from 'tsyringe'; +import _ from "lodash"; +import fs from "fs"; +import { readFile } from "fs/promises"; +import { container, singleton } from "tsyringe"; -import type { Browser, CookieParam, GoToOptions, Page, Viewport } from 'puppeteer'; -import type { Cookie } from 'set-cookie-parser'; -import puppeteer, { TimeoutError } from 'puppeteer'; +import type { + Browser, + CookieParam, + GoToOptions, + Page, + Viewport, +} from "puppeteer"; +import type { Cookie } from "set-cookie-parser"; +import puppeteer, { TimeoutError } from "puppeteer"; -import { Defer } from 'civkit/defer'; -import { AssertionFailureError, ParamValidationError } from 'civkit/civ-rpc'; -import { AsyncService } from 'civkit/async-service'; -import { FancyFile } from 'civkit/fancy-file'; -import { delay } from 'civkit/timeout'; +import { Defer } from "civkit/defer"; +import { AssertionFailureError, ParamValidationError } from "civkit/civ-rpc"; +import { AsyncService } from "civkit/async-service"; +import { FancyFile } from "civkit/fancy-file"; +import { delay } from "civkit/timeout"; -import { SecurityCompromiseError, ServiceCrashedError, ServiceNodeResourceDrainError } from '../../shared/lib/errors'; -import { CurlControl } from '../curl'; -import { AsyncLocalContext } from '../async-context'; -import { GlobalLogger } from '../logger'; -import { minimalStealth } from '../minimal-stealth'; -import { BlackHoleDetector } from '../blackhole-detector'; +import { + SecurityCompromiseError, + ServiceCrashedError, + ServiceNodeResourceDrainError, +} from "../../shared/lib/errors"; +import { CurlControl } from "../curl"; +import { AsyncLocalContext } from "../async-context"; +import { GlobalLogger } from "../logger"; +import { minimalStealth } from "../minimal-stealth"; +import { BlackHoleDetector } from "../blackhole-detector"; function getChromeLaunchArgs() { - const args = [ - '--disable-dev-shm-usage', - '--disable-blink-features=AutomationControlled', - ]; + const args = [ + "--disable-dev-shm-usage", + "--disable-blink-features=AutomationControlled", + ]; - // Docker and many CI kernels do not permit Chrome's sandbox namespaces. - if (process.env.PUPPETEER_DISABLE_SANDBOX !== 'false' && fs.existsSync('/.dockerenv')) { - args.push('--no-sandbox', '--disable-setuid-sandbox'); - } + // Docker and many CI kernels do not permit Chrome's sandbox namespaces. + if ( + process.env.PUPPETEER_DISABLE_SANDBOX !== "false" && + fs.existsSync("/.dockerenv") + ) { + args.push("--no-sandbox", "--disable-setuid-sandbox"); + } - return args; + return args; } - export interface ScrappingOptions { - proxyUrl?: string; - cookies?: Cookie[]; - overrideUserAgent?: string; - timeoutMs?: number; - locale?: string; - referer?: string; - extraHeaders?: Record; - viewport?: Viewport; - proxyResources?: boolean; - allocProxy?: string; + proxyUrl?: string; + cookies?: Cookie[]; + overrideUserAgent?: string; + timeoutMs?: number; + locale?: string; + referer?: string; + extraHeaders?: Record; + viewport?: Viewport; + proxyResources?: boolean; + allocProxy?: string; - sideLoad?: { - impersonate: { - [url: string]: { - status: number; - headers: { [k: string]: string | string[]; }; - contentType?: string; - body?: FancyFile; - }; - }; - proxyOrigin: { [origin: string]: string; }; + sideLoad?: { + impersonate: { + [url: string]: { + status: number; + headers: { [k: string]: string | string[] }; + contentType?: string; + body?: FancyFile; + }; }; - + proxyOrigin: { [origin: string]: string }; + }; } const SIMULATE_SCROLL = ` @@ -236,336 +247,401 @@ window.waitForSelector = waitForSelector; @singleton() export class SERPSpecializedPuppeteerControl extends AsyncService { + _sn = 0; + browser!: Browser; + logger = this.globalLogger.child({ service: this.constructor.name }); - _sn = 0; - browser!: Browser; - logger = this.globalLogger.child({ service: this.constructor.name }); + __loadedPage: Page[] = []; - __loadedPage: Page[] = []; + finalizerMap = new WeakMap>(); + snMap = new WeakMap(); + livePages = new Set(); + lastPageCratedAt: number = 0; + ua: string = ""; + effectiveUA: string = ""; - finalizerMap = new WeakMap>(); - snMap = new WeakMap(); - livePages = new Set(); - lastPageCratedAt: number = 0; - ua: string = ''; - effectiveUA: string = ''; + protected _REPORT_FUNCTION_NAME = "bingo"; - protected _REPORT_FUNCTION_NAME = 'bingo'; + lifeCycleTrack = new WeakMap(); - lifeCycleTrack = new WeakMap(); + constructor( + protected globalLogger: GlobalLogger, + protected asyncLocalContext: AsyncLocalContext, + protected curlControl: CurlControl, + protected blackHoleDetector: BlackHoleDetector, + ) { + super(...arguments); + this.setMaxListeners(Infinity); - constructor( - protected globalLogger: GlobalLogger, - protected asyncLocalContext: AsyncLocalContext, - protected curlControl: CurlControl, - protected blackHoleDetector: BlackHoleDetector, - ) { - super(...arguments); - this.setMaxListeners(Infinity); - - let crippledTimes = 0; - this.on('crippled', () => { - crippledTimes += 1; - this.__loadedPage.length = 0; - this.livePages.clear(); - if (crippledTimes > 5) { - process.nextTick(() => { - this.emit('error', new Error('Browser crashed too many times, quitting...')); - // process.exit(1); - }); - } + let crippledTimes = 0; + this.on("crippled", () => { + crippledTimes += 1; + this.__loadedPage.length = 0; + this.livePages.clear(); + if (crippledTimes > 5) { + process.nextTick(() => { + this.emit( + "error", + new Error("Browser crashed too many times, quitting..."), + ); + // process.exit(1); }); + } + }); + } + + override async init() { + await this.dependencyReady(); + if (process.env.NODE_ENV?.includes("dry-run")) { + this.emit("ready"); + return; } - override async init() { - await this.dependencyReady(); - if (process.env.NODE_ENV?.includes('dry-run')) { - this.emit('ready'); - return; - } + if (this.browser) { + if (this.browser.connected) { + await this.browser.close(); + } else { + this.browser.process()?.kill("SIGKILL"); + } + } + this.browser = await puppeteer + .launch({ + timeout: 30_000, + headless: !Boolean(process.env.DEBUG_BROWSER), + pipe: true, + executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH, + args: getChromeLaunchArgs(), + }) + .catch((err: any) => { + this.logger.error(`Unknown firebase issue, just die fast.`, { err }); + process.nextTick(() => { + this.emit("error", err); + // process.exit(1); + }); + return Promise.reject(err); + }); + this.browser.once("disconnected", () => { + this.logger.warn(`Browser disconnected`); + if (this.browser) { + this.emit("crippled"); + } + process.nextTick(() => this.serviceReady()); + }); + this.ua = await this.browser.userAgent(); + this.logger.info( + `Browser launched: ${this.browser.process()?.pid}, ${this.ua}`, + ); + this.effectiveUA = this.ua + .replace(/Headless/i, "") + .replace( + "Mozilla/5.0 (X11; Linux x86_64)", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + ); + this.curlControl.impersonateChrome(this.effectiveUA); - if (this.browser) { - if (this.browser.connected) { - await this.browser.close(); - } else { - this.browser.process()?.kill('SIGKILL'); - } - } - this.browser = await puppeteer.launch({ - timeout: 30_000, - headless: !Boolean(process.env.DEBUG_BROWSER), - pipe: true, - executablePath: process.env.OVERRIDE_CHROME_EXECUTABLE_PATH, - args: getChromeLaunchArgs(), - }).catch((err: any) => { - this.logger.error(`Unknown firebase issue, just die fast.`, { err }); - process.nextTick(() => { - this.emit('error', err); - // process.exit(1); + await this.newPage("beware_deadlock").then((r) => + this.__loadedPage.push(r), + ); + + this.emit("ready"); + } + + async newPage(bewareDeadLock: any = false) { + if (!bewareDeadLock) { + await this.serviceReady(); + } + const sn = this._sn++; + let page; + try { + const dedicatedContext = await this.browser.createBrowserContext(); + page = await dedicatedContext.newPage(); + } catch (err: any) { + this.logger.warn(`Failed to create page ${sn}`, { err }); + this.browser.process()?.kill("SIGKILL"); + throw new ServiceNodeResourceDrainError( + `This specific worker node failed to open a new page, try again.`, + ); + } + const preparations = []; + + preparations.push(page.setUserAgent(this.effectiveUA)); + // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); + // preparations.push(page.setUserAgent(`Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`)); + preparations.push(page.setBypassCSP(true)); + preparations.push(page.setViewport({ width: 1024, height: 1024 })); + preparations.push( + page.exposeFunction(this._REPORT_FUNCTION_NAME, (thing: T) => { + page.emit(this._REPORT_FUNCTION_NAME, thing); + }), + ); + preparations.push( + page.exposeFunction("setViewport", (viewport: Viewport | null) => { + page.setViewport(viewport).catch(() => undefined); + }), + ); + preparations.push(page.evaluateOnNewDocument(SCRIPT_TO_INJECT_INTO_FRAME)); + + await Promise.all(preparations); + + this.snMap.set(page, sn); + this.logger.debug(`Page ${sn} created.`); + this.lastPageCratedAt = Date.now(); + this.livePages.add(page); + + return page; + } + + async getNextPage() { + let thePage: Page | undefined; + if (this.__loadedPage.length) { + thePage = this.__loadedPage.shift(); + if (this.__loadedPage.length <= 1) { + process.nextTick(() => { + this.newPage() + .then((r) => this.__loadedPage.push(r)) + .catch((err) => { + this.logger.warn(`Failed to load new page ahead of time`, { + err, + }); }); - return Promise.reject(err); }); - this.browser.once('disconnected', () => { - this.logger.warn(`Browser disconnected`); - if (this.browser) { - this.emit('crippled'); - } - process.nextTick(() => this.serviceReady()); - }); - this.ua = await this.browser.userAgent(); - this.logger.info(`Browser launched: ${this.browser.process()?.pid}, ${this.ua}`); - this.effectiveUA = this.ua.replace(/Headless/i, '').replace('Mozilla/5.0 (X11; Linux x86_64)', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'); - this.curlControl.impersonateChrome(this.effectiveUA); - - await this.newPage('beware_deadlock').then((r) => this.__loadedPage.push(r)); - - this.emit('ready'); + } } - async newPage(bewareDeadLock: any = false) { - if (!bewareDeadLock) { - await this.serviceReady(); - } - const sn = this._sn++; - let page; + if (!thePage) { + thePage = await this.newPage(); + } + + const timer = setTimeout(() => { + this.logger.warn( + `Page is not allowed to live past 5 minutes, ditching page ${this.snMap.get(thePage!)}...`, + ); + this.ditchPage(thePage!); + }, 300 * 1000); + + this.finalizerMap.set(thePage, timer); + + return thePage; + } + + async ditchPage(page: Page) { + if (this.finalizerMap.has(page)) { + clearTimeout(this.finalizerMap.get(page)!); + this.finalizerMap.delete(page); + } + if (page.isClosed()) { + return; + } + const sn = this.snMap.get(page); + this.logger.debug(`Closing page ${sn}`); + await Promise.race([ + (async () => { + const ctx = page.browserContext(); try { - const dedicatedContext = await this.browser.createBrowserContext(); - page = await dedicatedContext.newPage(); - } catch (err: any) { - this.logger.warn(`Failed to create page ${sn}`, { err }); - this.browser.process()?.kill('SIGKILL'); - throw new ServiceNodeResourceDrainError(`This specific worker node failed to open a new page, try again.`); + await page.close(); + } finally { + await ctx.close(); } - const preparations = []; + })(), + delay(5000), + ]).catch((err) => { + this.logger.error(`Failed to destroy page ${sn}`, { err }); + }); + this.livePages.delete(page); + } - preparations.push(page.setUserAgent(this.effectiveUA)); - // preparations.push(page.setUserAgent(`Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)`)); - // preparations.push(page.setUserAgent(`Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)`)); - preparations.push(page.setBypassCSP(true)); - preparations.push(page.setViewport({ width: 1024, height: 1024 })); - preparations.push(page.exposeFunction(this._REPORT_FUNCTION_NAME, (thing: T) => { - page.emit(this._REPORT_FUNCTION_NAME, thing); - })); - preparations.push(page.exposeFunction('setViewport', (viewport: Viewport | null) => { - page.setViewport(viewport).catch(() => undefined); - })); - preparations.push(page.evaluateOnNewDocument(SCRIPT_TO_INJECT_INTO_FRAME)); + async controlledScrap( + parsedUrl: URL, + func: (this: void) => Promise, + options: ScrappingOptions = {}, + ): Promise { + // parsedUrl.search = ''; + const url = parsedUrl.toString(); + const page = await this.getNextPage(); + this.lifeCycleTrack.set(page, this.asyncLocalContext.ctx); + page.on("response", (_resp) => { + this.blackHoleDetector.itWorked(); + }); + page.on("request", async (req) => { + if (req.isInterceptResolutionHandled()) { + return; + } + const reqUrlParsed = new URL(req.url()); + if (!reqUrlParsed.protocol.startsWith("http")) { + const overrides = req.continueRequestOverrides(); - await Promise.all(preparations); - - this.snMap.set(page, sn); - this.logger.debug(`Page ${sn} created.`); - this.lastPageCratedAt = Date.now(); - this.livePages.add(page); - - return page; - } - - async getNextPage() { - let thePage: Page | undefined; - if (this.__loadedPage.length) { - thePage = this.__loadedPage.shift(); - if (this.__loadedPage.length <= 1) { - process.nextTick(() => { - this.newPage() - .then((r) => this.__loadedPage.push(r)) - .catch((err) => { - this.logger.warn(`Failed to load new page ahead of time`, { err }); - }); - }); - } - } - - if (!thePage) { - thePage = await this.newPage(); - } - - const timer = setTimeout(() => { - this.logger.warn(`Page is not allowed to live past 5 minutes, ditching page ${this.snMap.get(thePage!)}...`); - this.ditchPage(thePage!); - }, 300 * 1000); - - this.finalizerMap.set(thePage, timer); - - return thePage; - } - - async ditchPage(page: Page) { - if (this.finalizerMap.has(page)) { - clearTimeout(this.finalizerMap.get(page)!); - this.finalizerMap.delete(page); - } - if (page.isClosed()) { - return; - } - const sn = this.snMap.get(page); - this.logger.debug(`Closing page ${sn}`); - await Promise.race([ - (async () => { - const ctx = page.browserContext(); - try { - await page.close(); - } finally { - await ctx.close(); - } - })(), - delay(5000) - ]).catch((err) => { - this.logger.error(`Failed to destroy page ${sn}`, { err }); - }); - this.livePages.delete(page); - } - - async controlledScrap(parsedUrl: URL, func: (this: void) => Promise, options: ScrappingOptions = {}): Promise { - // parsedUrl.search = ''; - const url = parsedUrl.toString(); - const page = await this.getNextPage(); - this.lifeCycleTrack.set(page, this.asyncLocalContext.ctx); - page.on('response', (_resp) => { - this.blackHoleDetector.itWorked(); - }); - page.on('request', async (req) => { - if (req.isInterceptResolutionHandled()) { - return; - }; - const reqUrlParsed = new URL(req.url()); - if (!reqUrlParsed.protocol.startsWith('http')) { - const overrides = req.continueRequestOverrides(); - - return req.continue(overrides, 0); - } - const typ = req.resourceType(); - if (typ === 'media' || typ === 'font' || typ === 'image' || typ === 'stylesheet') { - // Non-cooperative answer to block all media requests. - return req.abort('blockedbyclient'); - } - if (!options.proxyResources) { - const isDocRequest = ['document', 'xhr', 'fetch', 'websocket', 'prefetch', 'eventsource', 'ping'].includes(typ); - if (!isDocRequest) { - if (options.extraHeaders) { - const overrides = req.continueRequestOverrides(); - const continueArgs = [{ - ...overrides, - headers: { - ...req.headers(), - ...overrides?.headers, - ...options.extraHeaders, - } - }, 1] as const; - - return req.continue(continueArgs[0], continueArgs[1]); - } - const overrides = req.continueRequestOverrides(); - - return req.continue(overrides, 0); - } - } - const sideload = options.sideLoad; - - const impersonate = sideload?.impersonate[reqUrlParsed.href]; - if (impersonate) { - let body; - if (impersonate.body) { - body = await readFile(await impersonate.body.filePath); - if (req.isInterceptResolutionHandled()) { - return; - } - } - return req.respond({ - status: impersonate.status, - headers: impersonate.headers, - contentType: impersonate.contentType, - body: body ? Uint8Array.from(body) : undefined, - }, 999); - } - - const proxy = options.proxyUrl || sideload?.proxyOrigin?.[reqUrlParsed.origin]; - const ctx = this.lifeCycleTrack.get(page); - if (proxy && ctx) { - return await this.asyncLocalContext.bridge(ctx, async () => { - try { - const curled = await this.curlControl.sideLoad(reqUrlParsed, { - ...options, - method: req.method(), - body: req.postData(), - extraHeaders: { - ...req.headers(), - ...options.extraHeaders, - }, - proxyUrl: proxy - }); - if (req.isInterceptResolutionHandled()) { - return; - }; - - if (curled.chain.length === 1) { - if (!curled.file) { - return req.respond({ - status: curled.status, - headers: _.omit(curled.headers, 'result'), - contentType: curled.contentType, - }, 3); - } - const body = await readFile(await curled.file.filePath); - if (req.isInterceptResolutionHandled()) { - return; - }; - return req.respond({ - status: curled.status, - headers: _.omit(curled.headers, 'result'), - contentType: curled.contentType, - body: Uint8Array.from(body), - }, 3); - } - options.sideLoad ??= curled.sideLoadOpts; - _.merge(options.sideLoad, curled.sideLoadOpts); - const firstReq = curled.chain[0]; - - return req.respond({ - status: (firstReq.result as any)!.code, - headers: _.omit(firstReq, 'result'), - }, 3); - } catch (err: any) { - this.logger.warn(`Failed to sideload browser request ${reqUrlParsed.origin}`, { href: reqUrlParsed.href, err, proxy }); - } - if (req.isInterceptResolutionHandled()) { - return; - }; - const overrides = req.continueRequestOverrides(); - const continueArgs = [{ - ...overrides, - headers: { - ...req.headers(), - ...overrides?.headers, - ...options.extraHeaders, - } - }, 1] as const; - - return req.continue(continueArgs[0], continueArgs[1]); - }); - } - - if (req.isInterceptResolutionHandled()) { - return; - }; + return req.continue(overrides, 0); + } + const typ = req.resourceType(); + if ( + typ === "media" || + typ === "font" || + typ === "image" || + typ === "stylesheet" + ) { + // Non-cooperative answer to block all media requests. + return req.abort("blockedbyclient"); + } + if (!options.proxyResources) { + const isDocRequest = [ + "document", + "xhr", + "fetch", + "websocket", + "prefetch", + "eventsource", + "ping", + ].includes(typ); + if (!isDocRequest) { + if (options.extraHeaders) { const overrides = req.continueRequestOverrides(); - const continueArgs = [{ + const continueArgs = [ + { ...overrides, headers: { - ...req.headers(), - ...overrides?.headers, - ...options.extraHeaders, - } - }, 1] as const; + ...req.headers(), + ...overrides?.headers, + ...options.extraHeaders, + }, + }, + 1, + ] as const; return req.continue(continueArgs[0], continueArgs[1]); + } + const overrides = req.continueRequestOverrides(); + + return req.continue(overrides, 0); + } + } + const sideload = options.sideLoad; + + const impersonate = sideload?.impersonate[reqUrlParsed.href]; + if (impersonate) { + let body; + if (impersonate.body) { + body = await readFile(await impersonate.body.filePath); + if (req.isInterceptResolutionHandled()) { + return; + } + } + return req.respond( + { + status: impersonate.status, + headers: impersonate.headers, + contentType: impersonate.contentType, + body: body ? Uint8Array.from(body) : undefined, + }, + 999, + ); + } + + const proxy = + options.proxyUrl || sideload?.proxyOrigin?.[reqUrlParsed.origin]; + const ctx = this.lifeCycleTrack.get(page); + if (proxy && ctx) { + return await this.asyncLocalContext.bridge(ctx, async () => { + try { + const curled = await this.curlControl.sideLoad(reqUrlParsed, { + ...options, + method: req.method(), + body: req.postData(), + extraHeaders: { + ...req.headers(), + ...options.extraHeaders, + }, + proxyUrl: proxy, + }); + if (req.isInterceptResolutionHandled()) { + return; + } + + if (curled.chain.length === 1) { + if (!curled.file) { + return req.respond( + { + status: curled.status, + headers: _.omit(curled.headers, "result"), + contentType: curled.contentType, + }, + 3, + ); + } + const body = await readFile(await curled.file.filePath); + if (req.isInterceptResolutionHandled()) { + return; + } + return req.respond( + { + status: curled.status, + headers: _.omit(curled.headers, "result"), + contentType: curled.contentType, + body: Uint8Array.from(body), + }, + 3, + ); + } + options.sideLoad ??= curled.sideLoadOpts; + _.merge(options.sideLoad, curled.sideLoadOpts); + const firstReq = curled.chain[0]; + + return req.respond( + { + status: (firstReq.result as any)!.code, + headers: _.omit(firstReq, "result"), + }, + 3, + ); + } catch (err: any) { + this.logger.warn( + `Failed to sideload browser request ${reqUrlParsed.origin}`, + { href: reqUrlParsed.href, err, proxy }, + ); + } + if (req.isInterceptResolutionHandled()) { + return; + } + const overrides = req.continueRequestOverrides(); + const continueArgs = [ + { + ...overrides, + headers: { + ...req.headers(), + ...overrides?.headers, + ...options.extraHeaders, + }, + }, + 1, + ] as const; + + return req.continue(continueArgs[0], continueArgs[1]); }); - await page.setRequestInterception(true); + } - const sn = this.snMap.get(page); - this.logger.info(`Page ${sn}: Scraping ${url}`, { url }); + if (req.isInterceptResolutionHandled()) { + return; + } + const overrides = req.continueRequestOverrides(); + const continueArgs = [ + { + ...overrides, + headers: { + ...req.headers(), + ...overrides?.headers, + ...options.extraHeaders, + }, + }, + 1, + ] as const; - await page.evaluateOnNewDocument(`(function () { + return req.continue(continueArgs[0], continueArgs[1]); + }); + await page.setRequestInterception(true); + + const sn = this.snMap.get(page); + this.logger.info(`Page ${sn}: Scraping ${url}`, { url }); + + await page.evaluateOnNewDocument(`(function () { if (window.top !== window.self) { return; } @@ -579,130 +655,132 @@ func().then((result) => { })();`); - if (options.locale) { - // Add headers via request interception to walk around this bug - // https://github.com/puppeteer/puppeteer/issues/10235 - // await page.setExtraHTTPHeaders({ - // 'Accept-Language': options.locale - // }); + if (options.locale) { + // Add headers via request interception to walk around this bug + // https://github.com/puppeteer/puppeteer/issues/10235 + // await page.setExtraHTTPHeaders({ + // 'Accept-Language': options.locale + // }); - await page.evaluateOnNewDocument(() => { - Object.defineProperty(navigator, "language", { - get: function () { - return options.locale; - } - }); - Object.defineProperty(navigator, "languages", { - get: function () { - return [options.locale]; - } - }); - }); - } - - if (options.cookies) { - const mapped = options.cookies.map((x) => { - const draft: CookieParam = { - name: x.name, - value: encodeURIComponent(x.value), - secure: x.secure, - domain: x.domain, - path: x.path, - expires: x.expires ? Math.floor(x.expires.valueOf() / 1000) : undefined, - sameSite: x.sameSite as any, - }; - if (!draft.expires && x.maxAge) { - draft.expires = Math.floor(Date.now() / 1000) + x.maxAge; - } - if (!draft.domain) { - draft.url = parsedUrl.toString(); - } - - return draft; - }); - try { - await page.setCookie(...mapped); - } catch (err: any) { - this.logger.warn(`Page ${sn}: Failed to set cookies`, { err }); - throw new ParamValidationError({ - path: 'cookies', - message: `Failed to set cookies: ${err?.message}` - }); - } - } - if (options.overrideUserAgent) { - await page.setUserAgent(options.overrideUserAgent); - } - if (options.viewport) { - await page.setViewport(options.viewport); - } - - const resultDeferred = Defer(); - const crippleListener = () => resultDeferred.reject(new ServiceCrashedError({ message: `Browser crashed, try again` })); - this.once('crippled', crippleListener); - resultDeferred.promise.finally(() => { - this.off('crippled', crippleListener); + await page.evaluateOnNewDocument(() => { + Object.defineProperty(navigator, "language", { + get: function () { + return options.locale; + }, }); - const hdl = (s: { - err?: any; - data?: T; - }) => { - if (s.err) { - resultDeferred.reject(s.err); - } - resultDeferred.resolve(s.data); - }; - page.on(this._REPORT_FUNCTION_NAME, hdl as any); - page.once('abuse', (event: any) => { - this.emit('abuse', { ...event, url: parsedUrl }); - - resultDeferred.reject( - new SecurityCompromiseError(`Abuse detected: ${event.reason}`) - ); + Object.defineProperty(navigator, "languages", { + get: function () { + return [options.locale]; + }, }); - - const timeout = options.timeoutMs || 45_000; - const goToOptions: GoToOptions = { - waitUntil: ['domcontentloaded', 'load'], - timeout, - }; - - if (options.referer) { - goToOptions.referer = options.referer; - } - - const gotoPromise = page.goto(url, goToOptions) - .catch((err) => { - if (err instanceof TimeoutError) { - this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err }); - throw new AssertionFailureError({ - message: `Failed to goto ${url}: ${err}`, - cause: err, - }); - } - - this.logger.warn(`Page ${sn}: Browsing of ${url} aborted`, { err }); - return undefined; - }); - - const controlTimeout = setTimeout(() => { - resultDeferred.reject(new TimeoutError(`Control function did not respond in time`)); - }, timeout); - controlTimeout.unref(); - - try { - return await Promise.race([ - resultDeferred.promise, - gotoPromise.then(() => resultDeferred.promise), - ]); - } finally { - clearTimeout(controlTimeout); - page.off(this._REPORT_FUNCTION_NAME, hdl as any); - this.ditchPage(page); - resultDeferred.resolve(); - } + }); } + if (options.cookies) { + const mapped = options.cookies.map((x) => { + const draft: CookieParam = { + name: x.name, + value: encodeURIComponent(x.value), + secure: x.secure, + domain: x.domain, + path: x.path, + expires: x.expires + ? Math.floor(x.expires.valueOf() / 1000) + : undefined, + sameSite: x.sameSite as any, + }; + if (!draft.expires && x.maxAge) { + draft.expires = Math.floor(Date.now() / 1000) + x.maxAge; + } + if (!draft.domain) { + draft.url = parsedUrl.toString(); + } + + return draft; + }); + try { + await page.setCookie(...mapped); + } catch (err: any) { + this.logger.warn(`Page ${sn}: Failed to set cookies`, { err }); + throw new ParamValidationError({ + path: "cookies", + message: `Failed to set cookies: ${err?.message}`, + }); + } + } + if (options.overrideUserAgent) { + await page.setUserAgent(options.overrideUserAgent); + } + if (options.viewport) { + await page.setViewport(options.viewport); + } + + const resultDeferred = Defer(); + const crippleListener = () => + resultDeferred.reject( + new ServiceCrashedError({ message: `Browser crashed, try again` }), + ); + this.once("crippled", crippleListener); + resultDeferred.promise.finally(() => { + this.off("crippled", crippleListener); + }); + const hdl = (s: { err?: any; data?: T }) => { + if (s.err) { + resultDeferred.reject(s.err); + } + resultDeferred.resolve(s.data); + }; + page.on(this._REPORT_FUNCTION_NAME, hdl as any); + page.once("abuse", (event: any) => { + this.emit("abuse", { ...event, url: parsedUrl }); + + resultDeferred.reject( + new SecurityCompromiseError(`Abuse detected: ${event.reason}`), + ); + }); + + const timeout = options.timeoutMs || 45_000; + const goToOptions: GoToOptions = { + waitUntil: ["domcontentloaded", "load"], + timeout, + }; + + if (options.referer) { + goToOptions.referer = options.referer; + } + + const gotoPromise = page.goto(url, goToOptions).catch((err) => { + if (err instanceof TimeoutError) { + this.logger.warn(`Page ${sn}: Browsing of ${url} timed out`, { err }); + throw new AssertionFailureError({ + message: `Failed to goto ${url}: ${err}`, + cause: err, + }); + } + + this.logger.warn(`Page ${sn}: Browsing of ${url} aborted`, { err }); + return undefined; + }); + + const controlTimeout = setTimeout(() => { + resultDeferred.reject( + new TimeoutError(`Control function did not respond in time`), + ); + }, timeout); + controlTimeout.unref(); + + try { + return await Promise.race([ + resultDeferred.promise, + gotoPromise.then(() => resultDeferred.promise), + ]); + } finally { + clearTimeout(controlTimeout); + page.off(this._REPORT_FUNCTION_NAME, hdl as any); + this.ditchPage(page); + resultDeferred.resolve(); + } + } } const puppeteerControl = container.resolve(SERPSpecializedPuppeteerControl); diff --git a/src/services/serp/serper.ts b/src/services/serp/serper.ts index 51317ea..589567e 100644 --- a/src/services/serp/serper.ts +++ b/src/services/serp/serper.ts @@ -1,165 +1,182 @@ - -import { singleton } from 'tsyringe'; -import { GlobalLogger } from '../logger'; -import { SecretExposer } from '../../shared/services/secrets'; -import { AsyncLocalContext } from '../async-context'; -import { SerperBingHTTP, SerperGoogleHTTP, SerperImageSearchResponse, SerperNewsSearchResponse, SerperSearchQueryParams, SerperWebSearchResponse } from '../../shared/3rd-party/serper-search'; -import { BlackHoleDetector } from '../blackhole-detector'; -import { Context } from '../registry'; -import { AsyncService } from 'civkit/async-service'; -import { AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from 'civkit/civ-rpc'; +import { singleton } from "tsyringe"; +import { GlobalLogger } from "../logger"; +import { SecretExposer } from "../../shared/services/secrets"; +import { AsyncLocalContext } from "../async-context"; +import { + SerperBingHTTP, + SerperGoogleHTTP, + SerperImageSearchResponse, + SerperNewsSearchResponse, + SerperSearchQueryParams, + SerperWebSearchResponse, +} from "../../shared/3rd-party/serper-search"; +import { BlackHoleDetector } from "../blackhole-detector"; +import { Context } from "../registry"; +import { AsyncService } from "civkit/async-service"; +import { AutoCastable, Prop, RPC_CALL_ENVIRONMENT } from "civkit/civ-rpc"; @singleton() export class SerperGoogleSearchService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + client!: SerperGoogleHTTP; - client!: SerperGoogleHTTP; + constructor( + protected globalLogger: GlobalLogger, + protected secretExposer: SecretExposer, + protected threadLocal: AsyncLocalContext, + protected blackHoleDetector: BlackHoleDetector, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected secretExposer: SecretExposer, - protected threadLocal: AsyncLocalContext, - protected blackHoleDetector: BlackHoleDetector, - ) { - super(...arguments); + override async init() { + await this.dependencyReady(); + this.emit("ready"); + + this.client = new SerperGoogleHTTP( + this.secretExposer.SERPER_SEARCH_API_KEY, + ); + } + + doSearch( + variant: "web", + query: SerperSearchQueryParams, + ): Promise; + doSearch( + variant: "images", + query: SerperSearchQueryParams, + ): Promise; + doSearch( + variant: "news", + query: SerperSearchQueryParams, + ): Promise; + async doSearch( + variant: "web" | "images" | "news", + query: SerperSearchQueryParams, + ) { + this.logger.debug(`Doing external search`, query); + let results; + switch (variant) { + case "images": { + const r = await this.client.imageSearch(query); + + results = r.parsed.images; + break; + } + case "news": { + const r = await this.client.newsSearch(query); + + results = r.parsed.news; + break; + } + case "web": + default: { + const r = await this.client.webSearch(query); + + results = r.parsed.organic; + break; + } } - override async init() { - await this.dependencyReady(); - this.emit('ready'); + this.blackHoleDetector.itWorked(); - this.client = new SerperGoogleHTTP(this.secretExposer.SERPER_SEARCH_API_KEY); - } - - - doSearch(variant: 'web', query: SerperSearchQueryParams): Promise; - doSearch(variant: 'images', query: SerperSearchQueryParams): Promise; - doSearch(variant: 'news', query: SerperSearchQueryParams): Promise; - async doSearch(variant: 'web' | 'images' | 'news', query: SerperSearchQueryParams) { - this.logger.debug(`Doing external search`, query); - let results; - switch (variant) { - case 'images': { - const r = await this.client.imageSearch(query); - - results = r.parsed.images; - break; - } - case 'news': { - const r = await this.client.newsSearch(query); - - results = r.parsed.news; - break; - } - case 'web': - default: { - const r = await this.client.webSearch(query); - - results = r.parsed.organic; - break; - } - } - - this.blackHoleDetector.itWorked(); - - return results; - } - - - async webSearch(query: SerperSearchQueryParams) { - return this.doSearch('web', query); - } - async imageSearch(query: SerperSearchQueryParams) { - return this.doSearch('images', query); - } - async newsSearch(query: SerperSearchQueryParams) { - return this.doSearch('news', query); - } + return results; + } + async webSearch(query: SerperSearchQueryParams) { + return this.doSearch("web", query); + } + async imageSearch(query: SerperSearchQueryParams) { + return this.doSearch("images", query); + } + async newsSearch(query: SerperSearchQueryParams) { + return this.doSearch("news", query); + } } @singleton() export class SerperBingSearchService extends SerperGoogleSearchService { - override client!: SerperBingHTTP; + override client!: SerperBingHTTP; - override async init() { - await this.dependencyReady(); - this.emit('ready'); + override async init() { + await this.dependencyReady(); + this.emit("ready"); - this.client = new SerperBingHTTP(this.secretExposer.SERPER_SEARCH_API_KEY); - } + this.client = new SerperBingHTTP(this.secretExposer.SERPER_SEARCH_API_KEY); + } } export class GoogleSearchExplicitOperatorsDto extends AutoCastable { - @Prop({ - arrayOf: String, - desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.` - }) - ext?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.`, + }) + ext?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.` - }) - filetype?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.`, + }) + filetype?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.` - }) - intitle?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.`, + }) + intitle?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.` - }) - loc?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`, + }) + loc?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.` - }) - site?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.`, + }) + site?: string | string[]; - addTo(searchTerm: string) { - const chunks = []; - for (const [key, value] of Object.entries(this)) { - if (value) { - const values = Array.isArray(value) ? value : [value]; - const textValue = values.map((v) => `${key}:${v}`).join(' OR '); - if (textValue) { - chunks.push(textValue); - } - } + addTo(searchTerm: string) { + const chunks = []; + for (const [key, value] of Object.entries(this)) { + if (value) { + const values = Array.isArray(value) ? value : [value]; + const textValue = values.map((v) => `${key}:${v}`).join(" OR "); + if (textValue) { + chunks.push(textValue); } - const opPart = chunks.length > 1 ? chunks.map((x) => `(${x})`).join(' AND ') : chunks; + } + } + const opPart = + chunks.length > 1 ? chunks.map((x) => `(${x})`).join(" AND ") : chunks; - if (opPart.length) { - return [searchTerm, opPart].join(' '); - } - - return searchTerm; + if (opPart.length) { + return [searchTerm, opPart].join(" "); } - static override from(input: any) { - const instance = super.from(input) as GoogleSearchExplicitOperatorsDto; - const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined; + return searchTerm; + } - const params = ['ext', 'filetype', 'intitle', 'loc', 'site']; + static override from(input: any) { + const instance = super.from(input) as GoogleSearchExplicitOperatorsDto; + const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined; - for (const p of params) { - const customValue = ctx?.get(`x-${p}`) || ctx?.get(`${p}`); - if (!customValue) { - continue; - } + const params = ["ext", "filetype", "intitle", "loc", "site"]; - const filtered = customValue.split(', ').filter(Boolean); - if (filtered.length) { - Reflect.set(instance, p, filtered); - } - } + for (const p of params) { + const customValue = ctx?.get(`x-${p}`) || ctx?.get(`${p}`); + if (!customValue) { + continue; + } - return instance; + const filtered = customValue.split(", ").filter(Boolean); + if (filtered.length) { + Reflect.set(instance, p, filtered); + } } + + return instance; + } } diff --git a/src/services/serp/standalone-fallback.ts b/src/services/serp/standalone-fallback.ts index 9994fc7..e508cee 100644 --- a/src/services/serp/standalone-fallback.ts +++ b/src/services/serp/standalone-fallback.ts @@ -1,110 +1,135 @@ -import { singleton } from 'tsyringe'; -import { AsyncService } from 'civkit/async-service'; -import { GlobalLogger } from '../logger'; -import { JSDomControl } from '../jsdom'; -import { WebSearchEntry } from './compat'; -import { ServiceBadApproachError, ServiceBadAttemptError } from '../errors'; +import { singleton } from "tsyringe"; +import { AsyncService } from "civkit/async-service"; +import { GlobalLogger } from "../logger"; +import { JSDomControl } from "../jsdom"; +import { WebSearchEntry } from "./compat"; +import { ServiceBadApproachError, ServiceBadAttemptError } from "../errors"; function decodeDuckDuckGoTarget(href: string) { - try { - const url = new URL(href, 'https://html.duckduckgo.com'); - const actual = url.searchParams.get('uddg'); - if (actual) { - return decodeURIComponent(actual); - } - return url.toString(); - } catch { - return href; + try { + const url = new URL(href, "https://html.duckduckgo.com"); + const actual = url.searchParams.get("uddg"); + if (actual) { + return decodeURIComponent(actual); } + return url.toString(); + } catch { + return href; + } } function isAdLikeDuckDuckGoTarget(target: string) { - try { - const url = new URL(target); - if (url.hostname === 'duckduckgo.com' && url.pathname === '/y.js') { - return true; - } - return url.searchParams.has('ad_domain') || url.searchParams.has('ad_provider'); - } catch { - return /ad_domain=|ad_provider=/.test(target); + try { + const url = new URL(target); + if (url.hostname === "duckduckgo.com" && url.pathname === "/y.js") { + return true; } + return ( + url.searchParams.has("ad_domain") || url.searchParams.has("ad_provider") + ); + } catch { + return /ad_domain=|ad_provider=/.test(target); + } } @singleton() export class StandaloneSearchFallbackService extends AsyncService { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - constructor( - protected globalLogger: GlobalLogger, - protected jsDomControl: JSDomControl, - ) { - super(...arguments); + constructor( + protected globalLogger: GlobalLogger, + protected jsDomControl: JSDomControl, + ) { + super(...arguments); + } + + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } + + async webSearch(query: { [k: string]: any }) { + const url = new URL("https://html.duckduckgo.com/html/"); + url.searchParams.set("q", query.q || ""); + + const response = await fetch(url, { + headers: { + "user-agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36", + "accept-language": query.hl || "en-US,en;q=0.9", + }, + signal: AbortSignal.timeout(20_000), + }).catch((err) => { + throw new ServiceBadAttemptError( + `DuckDuckGo HTML fallback failed: ${err instanceof Error ? err.message : err}`, + ); + }); + + if (!response.ok) { + throw new ServiceBadAttemptError( + `DuckDuckGo HTML fallback returned ${response.status}`, + ); } - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } + const html = await response.text(); + const { document } = this.jsDomControl.linkedom.parseHTML(html).window; + const nodes = Array.from( + document.querySelectorAll(".result:not(.result--ad)"), + ); + const results = nodes + .map((node) => { + const linkElem = node.querySelector( + ".result__title a[href], .result__a[href]", + ); + const title = linkElem?.textContent?.replace(/\s+/g, " ").trim(); + const href = linkElem?.getAttribute("href"); - async webSearch(query: { [k: string]: any; }) { - const url = new URL('https://html.duckduckgo.com/html/'); - url.searchParams.set('q', query.q || ''); - - const response = await fetch(url, { - headers: { - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36', - 'accept-language': query.hl || 'en-US,en;q=0.9', - }, - signal: AbortSignal.timeout(20_000), - }).catch((err) => { - throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback failed: ${err instanceof Error ? err.message : err}`); - }); - - if (!response.ok) { - throw new ServiceBadAttemptError(`DuckDuckGo HTML fallback returned ${response.status}`); + if (!title || !href) { + return undefined; } - const html = await response.text(); - const { document } = this.jsDomControl.linkedom.parseHTML(html).window; - const nodes = Array.from(document.querySelectorAll('.result:not(.result--ad)')); - const results = nodes.map((node) => { - const linkElem = node.querySelector('.result__title a[href], .result__a[href]'); - const title = linkElem?.textContent?.replace(/\s+/g, ' ').trim(); - const href = linkElem?.getAttribute('href'); - - if (!title || !href) { - return undefined; - } - - const link = decodeDuckDuckGoTarget(href); - if (isAdLikeDuckDuckGoTarget(link)) { - return undefined; - } - - const snippet = node.querySelector('.result__snippet, .result-snippet')?.textContent?.replace(/\s+/g, ' ').trim(); - const source = node.querySelector('.result__url, .result__extras__url')?.textContent?.replace(/\s+/g, ' ').trim(); - - return { - link, - title, - snippet: snippet || undefined, - source: source || undefined, - variant: 'web', - } as WebSearchEntry; - }).filter(Boolean) as WebSearchEntry[]; - - if (!results.length) { - throw new ServiceBadAttemptError('DuckDuckGo HTML fallback returned no results.'); + const link = decodeDuckDuckGoTarget(href); + if (isAdLikeDuckDuckGoTarget(link)) { + return undefined; } - return results.slice(0, query.num || 10); + const snippet = node + .querySelector(".result__snippet, .result-snippet") + ?.textContent?.replace(/\s+/g, " ") + .trim(); + const source = node + .querySelector(".result__url, .result__extras__url") + ?.textContent?.replace(/\s+/g, " ") + .trim(); + + return { + link, + title, + snippet: snippet || undefined, + source: source || undefined, + variant: "web", + } as WebSearchEntry; + }) + .filter(Boolean) as WebSearchEntry[]; + + if (!results.length) { + throw new ServiceBadAttemptError( + "DuckDuckGo HTML fallback returned no results.", + ); } - async newsSearch() { - throw new ServiceBadApproachError('Standalone HTML fallback does not support news search.'); - } + return results.slice(0, query.num || 10); + } - async imageSearch() { - throw new ServiceBadApproachError('Standalone HTML fallback does not support image search.'); - } + async newsSearch() { + throw new ServiceBadApproachError( + "Standalone HTML fallback does not support news search.", + ); + } + + async imageSearch() { + throw new ServiceBadApproachError( + "Standalone HTML fallback does not support image search.", + ); + } } diff --git a/src/services/serper-search.ts b/src/services/serper-search.ts index 59698de..76599e5 100644 --- a/src/services/serper-search.ts +++ b/src/services/serper-search.ts @@ -1,212 +1,251 @@ -import { AsyncService, AutoCastable, DownstreamServiceFailureError, Prop, RPC_CALL_ENVIRONMENT, delay, marshalErrorLike } from 'civkit'; -import { singleton } from 'tsyringe'; -import { GlobalLogger } from './logger'; -import { SecretExposer } from '../shared/services/secrets'; -import { AsyncLocalContext } from './async-context'; -import { SerperBingHTTP, SerperGoogleHTTP, SerperImageSearchResponse, SerperNewsSearchResponse, SerperSearchQueryParams, SerperWebSearchResponse } from '../shared/3rd-party/serper-search'; -import { BlackHoleDetector } from './blackhole-detector'; -import { Context } from './registry'; -import { ServiceBadAttemptError } from '../shared'; +import { + AsyncService, + AutoCastable, + DownstreamServiceFailureError, + Prop, + RPC_CALL_ENVIRONMENT, + delay, + marshalErrorLike, +} from "civkit"; +import { singleton } from "tsyringe"; +import { GlobalLogger } from "./logger"; +import { SecretExposer } from "../shared/services/secrets"; +import { AsyncLocalContext } from "./async-context"; +import { + SerperBingHTTP, + SerperGoogleHTTP, + SerperImageSearchResponse, + SerperNewsSearchResponse, + SerperSearchQueryParams, + SerperWebSearchResponse, +} from "../shared/3rd-party/serper-search"; +import { BlackHoleDetector } from "./blackhole-detector"; +import { Context } from "./registry"; +import { ServiceBadAttemptError } from "../shared"; @singleton() export class SerperSearchService extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + serperGoogleSearchHTTP!: SerperGoogleHTTP; + serperBingSearchHTTP!: SerperBingHTTP; - serperGoogleSearchHTTP!: SerperGoogleHTTP; - serperBingSearchHTTP!: SerperBingHTTP; + constructor( + protected globalLogger: GlobalLogger, + protected secretExposer: SecretExposer, + protected threadLocal: AsyncLocalContext, + protected blackHoleDetector: BlackHoleDetector, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected secretExposer: SecretExposer, - protected threadLocal: AsyncLocalContext, - protected blackHoleDetector: BlackHoleDetector, - ) { - super(...arguments); + override async init() { + await this.dependencyReady(); + this.emit("ready"); + + this.serperGoogleSearchHTTP = new SerperGoogleHTTP( + this.secretExposer.SERPER_SEARCH_API_KEY, + ); + this.serperBingSearchHTTP = new SerperBingHTTP( + this.secretExposer.SERPER_SEARCH_API_KEY, + ); + } + + *iterClient() { + const preferBingSearch = this.threadLocal.get("bing-preferred"); + if (preferBingSearch) { + yield this.serperBingSearchHTTP; + } + while (true) { + yield this.serperGoogleSearchHTTP; + } + } + + doSearch( + variant: "web", + query: SerperSearchQueryParams, + ): Promise; + doSearch( + variant: "images", + query: SerperSearchQueryParams, + ): Promise; + doSearch( + variant: "news", + query: SerperSearchQueryParams, + ): Promise; + async doSearch( + variant: "web" | "images" | "news", + query: SerperSearchQueryParams, + ) { + const clientIt = this.iterClient(); + let client = clientIt.next().value; + if (!client) { + throw new Error(`Error iterating serper client`); } - override async init() { - await this.dependencyReady(); - this.emit('ready'); + let maxTries = 3; - this.serperGoogleSearchHTTP = new SerperGoogleHTTP(this.secretExposer.SERPER_SEARCH_API_KEY); - this.serperBingSearchHTTP = new SerperBingHTTP(this.secretExposer.SERPER_SEARCH_API_KEY); - } - - *iterClient() { - const preferBingSearch = this.threadLocal.get('bing-preferred'); - if (preferBingSearch) { - yield this.serperBingSearchHTTP; - } - while (true) { - yield this.serperGoogleSearchHTTP; - } - } - - doSearch(variant: 'web', query: SerperSearchQueryParams): Promise; - doSearch(variant: 'images', query: SerperSearchQueryParams): Promise; - doSearch(variant: 'news', query: SerperSearchQueryParams): Promise; - async doSearch(variant: 'web' | 'images' | 'news', query: SerperSearchQueryParams) { - const clientIt = this.iterClient(); - let client = clientIt.next().value; - if (!client) { - throw new Error(`Error iterating serper client`); - } - - let maxTries = 3; - - while (maxTries--) { - const t0 = Date.now(); - try { - this.logger.debug(`Doing external search`, query); - let r; - switch (variant) { - case 'images': { - r = await client.imageSearch(query); - const nextClient = clientIt.next().value; - if (nextClient && nextClient !== client) { - const results = r.parsed.images; - if (!results.length) { - client = nextClient; - throw new ServiceBadAttemptError('No results found'); - } - } - - break; - } - case 'news': { - r = await client.newsSearch(query); - const nextClient = clientIt.next().value; - if (nextClient && nextClient !== client) { - const results = r.parsed.news; - if (!results.length) { - client = nextClient; - throw new ServiceBadAttemptError('No results found'); - } - } - - break; - } - case 'web': - default: { - r = await client.webSearch(query); - const nextClient = clientIt.next().value; - if (nextClient && nextClient !== client) { - const results = r.parsed.organic; - if (!results.length) { - client = nextClient; - throw new ServiceBadAttemptError('No results found'); - } - } - - break; - } - } - const dt = Date.now() - t0; - this.blackHoleDetector.itWorked(); - this.logger.debug(`External search took ${dt}ms`, { searchDt: dt, variant }); - - return r.parsed; - } catch (err: any) { - const dt = Date.now() - t0; - this.logger.error(`${variant} search failed: ${err?.message}`, { searchDt: dt, err: marshalErrorLike(err) }); - if (err?.status === 429) { - await delay(500 + 1000 * Math.random()); - continue; - } - if (err instanceof ServiceBadAttemptError) { - continue; - } - - throw new DownstreamServiceFailureError({ message: `Search(${variant}) failed` }); + while (maxTries--) { + const t0 = Date.now(); + try { + this.logger.debug(`Doing external search`, query); + let r; + switch (variant) { + case "images": { + r = await client.imageSearch(query); + const nextClient = clientIt.next().value; + if (nextClient && nextClient !== client) { + const results = r.parsed.images; + if (!results.length) { + client = nextClient; + throw new ServiceBadAttemptError("No results found"); + } } + + break; + } + case "news": { + r = await client.newsSearch(query); + const nextClient = clientIt.next().value; + if (nextClient && nextClient !== client) { + const results = r.parsed.news; + if (!results.length) { + client = nextClient; + throw new ServiceBadAttemptError("No results found"); + } + } + + break; + } + case "web": + default: { + r = await client.webSearch(query); + const nextClient = clientIt.next().value; + if (nextClient && nextClient !== client) { + const results = r.parsed.organic; + if (!results.length) { + client = nextClient; + throw new ServiceBadAttemptError("No results found"); + } + } + + break; + } + } + const dt = Date.now() - t0; + this.blackHoleDetector.itWorked(); + this.logger.debug(`External search took ${dt}ms`, { + searchDt: dt, + variant, + }); + + return r.parsed; + } catch (err: any) { + const dt = Date.now() - t0; + this.logger.error(`${variant} search failed: ${err?.message}`, { + searchDt: dt, + err: marshalErrorLike(err), + }); + if (err?.status === 429) { + await delay(500 + 1000 * Math.random()); + continue; + } + if (err instanceof ServiceBadAttemptError) { + continue; } - throw new DownstreamServiceFailureError({ message: `Search(${variant}) failed` }); + throw new DownstreamServiceFailureError({ + message: `Search(${variant}) failed`, + }); + } } + throw new DownstreamServiceFailureError({ + message: `Search(${variant}) failed`, + }); + } - async webSearch(query: SerperSearchQueryParams) { - return this.doSearch('web', query); - } - async imageSearch(query: SerperSearchQueryParams) { - return this.doSearch('images', query); - } - async newsSearch(query: SerperSearchQueryParams) { - return this.doSearch('news', query); - } - + async webSearch(query: SerperSearchQueryParams) { + return this.doSearch("web", query); + } + async imageSearch(query: SerperSearchQueryParams) { + return this.doSearch("images", query); + } + async newsSearch(query: SerperSearchQueryParams) { + return this.doSearch("news", query); + } } export class GoogleSearchExplicitOperatorsDto extends AutoCastable { - @Prop({ - arrayOf: String, - desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.` - }) - ext?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages with a specific file extension. Example: to find the Honda GX120 Owner’s manual in PDF, type “Honda GX120 ownners manual ext:pdf”.`, + }) + ext?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.` - }) - filetype?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages created in the specified file type. Example: to find a web page created in PDF format about the evaluation of age-related cognitive changes, type “evaluation of age cognitive changes filetype:pdf”.`, + }) + filetype?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.` - }) - intitle?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns webpages containing the specified term in the title of the page. Example: to find pages about SEO conferences making sure the results contain 2023 in the title, type “seo conference intitle:2023”.`, + }) + intitle?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.` - }) - loc?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages written in the specified language. The language code must be in the ISO 639-1 two-letter code format. Example: to find information on visas only in Spanish, type “visas lang:es”.`, + }) + loc?: string | string[]; - @Prop({ - arrayOf: String, - desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.` - }) - site?: string | string[]; + @Prop({ + arrayOf: String, + desc: `Returns web pages coming only from a specific web site. Example: to find information about Goggles only on Brave pages, type “goggles site:brave.com”.`, + }) + site?: string | string[]; - addTo(searchTerm: string) { - const chunks = []; - for (const [key, value] of Object.entries(this)) { - if (value) { - const values = Array.isArray(value) ? value : [value]; - const textValue = values.map((v) => `${key}:${v}`).join(' OR '); - if (textValue) { - chunks.push(textValue); - } - } + addTo(searchTerm: string) { + const chunks = []; + for (const [key, value] of Object.entries(this)) { + if (value) { + const values = Array.isArray(value) ? value : [value]; + const textValue = values.map((v) => `${key}:${v}`).join(" OR "); + if (textValue) { + chunks.push(textValue); } - const opPart = chunks.length > 1 ? chunks.map((x) => `(${x})`).join(' AND ') : chunks; + } + } + const opPart = + chunks.length > 1 ? chunks.map((x) => `(${x})`).join(" AND ") : chunks; - if (opPart.length) { - return [searchTerm, opPart].join(' '); - } - - return searchTerm; + if (opPart.length) { + return [searchTerm, opPart].join(" "); } - static override from(input: any) { - const instance = super.from(input) as GoogleSearchExplicitOperatorsDto; - const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined; + return searchTerm; + } - const params = ['ext', 'filetype', 'intitle', 'loc', 'site']; + static override from(input: any) { + const instance = super.from(input) as GoogleSearchExplicitOperatorsDto; + const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined; - for (const p of params) { - const customValue = ctx?.get(`x-${p}`) || ctx?.get(`${p}`); - if (!customValue) { - continue; - } + const params = ["ext", "filetype", "intitle", "loc", "site"]; - const filtered = customValue.split(', ').filter(Boolean); - if (filtered.length) { - Reflect.set(instance, p, filtered); - } - } + for (const p of params) { + const customValue = ctx?.get(`x-${p}`) || ctx?.get(`${p}`); + if (!customValue) { + continue; + } - return instance; + const filtered = customValue.split(", ").filter(Boolean); + if (filtered.length) { + Reflect.set(instance, p, filtered); + } } + + return instance; + } } diff --git a/src/services/snapshot-formatter.ts b/src/services/snapshot-formatter.ts index 17b2570..bc24791 100644 --- a/src/services/snapshot-formatter.ts +++ b/src/services/snapshot-formatter.ts @@ -1,874 +1,1112 @@ -import { randomUUID } from 'crypto'; -import { container, singleton } from 'tsyringe'; -import { AssertionFailureError, AsyncService, DataStreamBrokenError, FancyFile, HashManager, marshalErrorLike } from 'civkit'; -import TurndownService, { Filter, Rule } from 'turndown'; -import { GlobalLogger } from './logger'; -import { PageSnapshot } from './puppeteer'; -import { FirebaseStorageBucketControl } from '../shared/services/firebase-storage-bucket'; -import { AsyncContext } from '../shared/services/async-context'; -import { Threaded } from '../services/threaded'; -import { JSDomControl } from './jsdom'; -import { AltTextService } from './alt-text'; -import { PDFExtractor } from './pdf-extract'; -import { cleanAttribute } from '../utils/misc'; -import _ from 'lodash'; -import { STATUS_CODES } from 'http'; -import type { CrawlerOptions } from '../dto/crawler-options'; -import { readFile } from '../utils/encoding'; -import { pathToFileURL } from 'url'; -import { countGPTToken } from '../shared/utils/openai'; - +import { randomUUID } from "crypto"; +import { container, singleton } from "tsyringe"; +import { + AssertionFailureError, + AsyncService, + DataStreamBrokenError, + FancyFile, + HashManager, + marshalErrorLike, +} from "civkit"; +import TurndownService, { Filter, Rule } from "turndown"; +import { GlobalLogger } from "./logger"; +import { PageSnapshot } from "./puppeteer"; +import { FirebaseStorageBucketControl } from "../shared/services/firebase-storage-bucket"; +import { AsyncContext } from "../shared/services/async-context"; +import { Threaded } from "../services/threaded"; +import { JSDomControl } from "./jsdom"; +import { AltTextService } from "./alt-text"; +import { PDFExtractor } from "./pdf-extract"; +import { cleanAttribute } from "../utils/misc"; +import _ from "lodash"; +import { STATUS_CODES } from "http"; +import type { CrawlerOptions } from "../dto/crawler-options"; +import { readFile } from "../utils/encoding"; +import { pathToFileURL } from "url"; +import { countGPTToken } from "../shared/utils/openai"; export interface FormattedPage { - title?: string; - description?: string; - url?: string; - content?: string; - publishedTime?: string; - html?: string; - text?: string; - screenshotUrl?: string; - screenshot?: Buffer; - pageshotUrl?: string; - pageshot?: Buffer; - links?: { [k: string]: string; } | [string, string][]; - images?: { [k: string]: string; } | [string, string][]; - warning?: string; - usage?: { - total_tokens?: number; - totalTokens?: number; - tokens?: number; - }; + title?: string; + description?: string; + url?: string; + content?: string; + publishedTime?: string; + html?: string; + text?: string; + screenshotUrl?: string; + screenshot?: Buffer; + pageshotUrl?: string; + pageshot?: Buffer; + links?: { [k: string]: string } | [string, string][]; + images?: { [k: string]: string } | [string, string][]; + warning?: string; + usage?: { + total_tokens?: number; + totalTokens?: number; + tokens?: number; + }; - textRepresentation?: string; + textRepresentation?: string; - [Symbol.dispose]?: () => void; + [Symbol.dispose]?: () => void; } -export const md5Hasher = new HashManager('md5', 'hex'); +export const md5Hasher = new HashManager("md5", "hex"); -const gfmPlugin = require('turndown-plugin-gfm'); +const gfmPlugin = require("turndown-plugin-gfm"); const highlightRegExp = /highlight-(?:text|source)-([a-z0-9]+)/; -const disallowedSummaryLinkSchemes = ['file:', 'javascript:', 'data:', 'vbscript:']; +const disallowedSummaryLinkSchemes = [ + "file:", + "javascript:", + "data:", + "vbscript:", +]; function hasDisallowedSummaryLinkScheme(href: string) { - const normalized = href.trim().toLowerCase(); + const normalized = href.trim().toLowerCase(); - return disallowedSummaryLinkSchemes.some((scheme) => normalized.startsWith(scheme)); + return disallowedSummaryLinkSchemes.some((scheme) => + normalized.startsWith(scheme), + ); } function escapeMarkdownTitleAttribute(title: string) { - return title.replace(/[\\"]/g, '\\$&'); + return title.replace(/[\\"]/g, "\\$&"); } export function highlightedCodeBlock(turndownService: TurndownService) { - turndownService.addRule('highlightedCodeBlock', { - filter: (node) => { - return ( - node.nodeName === 'DIV' && - node.firstChild?.nodeName === 'PRE' && - highlightRegExp.test(node.className) - ); - }, - replacement: (_content, node, options) => { - const className = (node as any).className || ''; - const language = (className.match(highlightRegExp) || [null, ''])[1]; + turndownService.addRule("highlightedCodeBlock", { + filter: (node) => { + return ( + node.nodeName === "DIV" && + node.firstChild?.nodeName === "PRE" && + highlightRegExp.test(node.className) + ); + }, + replacement: (_content, node, options) => { + const className = (node as any).className || ""; + const language = (className.match(highlightRegExp) || [null, ""])[1]; - return ( - '\n\n' + options.fence + language + '\n' + - node.firstChild!.textContent + - '\n' + options.fence + '\n\n' - ); - } - }); + return ( + "\n\n" + + options.fence + + language + + "\n" + + node.firstChild!.textContent + + "\n" + + options.fence + + "\n\n" + ); + }, + }); } @singleton() export class SnapshotFormatter extends AsyncService { + logger = this.globalLogger.child({ service: this.constructor.name }); - logger = this.globalLogger.child({ service: this.constructor.name }); + gfmPlugin = [ + gfmPlugin.tables, + highlightedCodeBlock, + gfmPlugin.strikethrough, + gfmPlugin.taskListItems, + ]; + gfmNoTable = [ + highlightedCodeBlock, + gfmPlugin.strikethrough, + gfmPlugin.taskListItems, + ]; - gfmPlugin = [gfmPlugin.tables, highlightedCodeBlock, gfmPlugin.strikethrough, gfmPlugin.taskListItems]; - gfmNoTable = [highlightedCodeBlock, gfmPlugin.strikethrough, gfmPlugin.taskListItems]; + constructor( + protected globalLogger: GlobalLogger, + protected jsdomControl: JSDomControl, + protected altTextService: AltTextService, + protected pdfExtractor: PDFExtractor, + protected threadLocal: AsyncContext, + protected firebaseObjectStorage: FirebaseStorageBucketControl, + ) { + super(...arguments); + } - constructor( - protected globalLogger: GlobalLogger, - protected jsdomControl: JSDomControl, - protected altTextService: AltTextService, - protected pdfExtractor: PDFExtractor, - protected threadLocal: AsyncContext, - protected firebaseObjectStorage: FirebaseStorageBucketControl, + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } + + @Threaded() + async formatSnapshot( + mode: string | "markdown" | "html" | "text" | "screenshot" | "pageshot", + snapshot: PageSnapshot & { + screenshotUrl?: string; + pageshotUrl?: string; + }, + nominalUrl?: URL, + urlValidMs = 3600 * 1000 * 4, + ) { + const t0 = Date.now(); + const f = { + ...(await this.getGeneralSnapshotMixins(snapshot)), + }; + let modeOK = false; + + if (mode.includes("screenshot")) { + modeOK = true; + if (snapshot.screenshot && !snapshot.screenshotUrl) { + const fid = `instant-screenshots/${randomUUID()}`; + await this.firebaseObjectStorage.saveFile(fid, snapshot.screenshot, { + metadata: { + contentType: "image/png", + }, + }); + snapshot.screenshotUrl = + await this.firebaseObjectStorage.signDownloadUrl( + fid, + Date.now() + urlValidMs, + ); + } + Object.assign(f, { + screenshotUrl: snapshot.screenshotUrl, + }); + + Object.defineProperty(f, "textRepresentation", { + value: `${f.screenshotUrl}\n`, + enumerable: false, + configurable: true, + }); + } + if (mode.includes("pageshot")) { + modeOK = true; + if (snapshot.pageshot && !snapshot.pageshotUrl) { + const fid = `instant-screenshots/${randomUUID()}`; + await this.firebaseObjectStorage.saveFile(fid, snapshot.pageshot, { + metadata: { + contentType: "image/png", + }, + }); + snapshot.pageshotUrl = await this.firebaseObjectStorage.signDownloadUrl( + fid, + Date.now() + urlValidMs, + ); + } + Object.assign(f, { + html: snapshot.html, + pageshotUrl: snapshot.pageshotUrl, + }); + Object.defineProperty(f, "textRepresentation", { + value: `${f.pageshotUrl}\n`, + enumerable: false, + configurable: true, + }); + } + if (mode.includes("html")) { + modeOK = true; + Object.assign(f, { + html: snapshot.html, + }); + + Object.defineProperty(f, "textRepresentation", { + value: snapshot.html, + enumerable: false, + configurable: true, + }); + } + + let pdfMode = false; + // in case of Google Web Cache content + if ( + snapshot.pdfs?.length && + (!snapshot.title || snapshot.title.startsWith("cache:")) ) { - super(...arguments); + const pdf = await this.pdfExtractor.cachedExtract( + snapshot.pdfs[0], + this.threadLocal.get("cacheTolerance"), + snapshot.pdfs[0].startsWith("http") ? undefined : snapshot.href, + ); + if (pdf) { + pdfMode = true; + snapshot.title = pdf.meta?.Title; + snapshot.text = pdf.text || snapshot.text; + snapshot.parsed = { + content: pdf.content, + textContent: pdf.content, + length: pdf.content?.length, + byline: pdf.meta?.Author, + lang: pdf.meta?.Language || undefined, + title: pdf.meta?.Title, + publishedTime: this.pdfExtractor + .parsePdfDate(pdf.meta?.ModDate || pdf.meta?.CreationDate) + ?.toISOString(), + }; + } } - override async init() { - await this.dependencyReady(); - this.emit('ready'); + if (mode.includes("text")) { + modeOK = true; + Object.assign(f, { + text: snapshot.text, + }); + Object.defineProperty(f, "textRepresentation", { + value: snapshot.text, + enumerable: false, + configurable: true, + }); } + if (mode.includes("lm")) { + modeOK = true; + f.content = snapshot.parsed?.textContent; + } - @Threaded() - async formatSnapshot(mode: string | 'markdown' | 'html' | 'text' | 'screenshot' | 'pageshot', snapshot: PageSnapshot & { - screenshotUrl?: string; - pageshotUrl?: string; - }, nominalUrl?: URL, urlValidMs = 3600 * 1000 * 4) { - const t0 = Date.now(); - const f = { - ...(await this.getGeneralSnapshotMixins(snapshot)), - }; - let modeOK = false; + if ( + modeOK && + (mode.includes("lm") || + (!mode.includes("markdown") && !mode.includes("content"))) + ) { + const dt = Date.now() - t0; + this.logger.debug(`Formatting took ${dt}ms`, { + mode, + url: nominalUrl?.toString(), + dt, + }); - if (mode.includes('screenshot')) { - modeOK = true; - if (snapshot.screenshot && !snapshot.screenshotUrl) { - const fid = `instant-screenshots/${randomUUID()}`; - await this.firebaseObjectStorage.saveFile(fid, snapshot.screenshot, { - metadata: { - contentType: 'image/png', - } - }); - snapshot.screenshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs); + const formatted: FormattedPage = { + title: (snapshot.parsed?.title || snapshot.title || "").trim(), + description: (snapshot.description || "").trim(), + url: nominalUrl?.toString() || snapshot.href?.trim(), + publishedTime: snapshot.parsed?.publishedTime || undefined, + }; + + Object.assign(f, formatted); + + return f; + } + + const imgDataUrlToObjectUrl = !Boolean( + this.threadLocal.get("keepImgDataUrl"), + ); + + let contentText = ""; + const imageSummary = {} as { [k: string]: string }; + const imageIdxTrack = new Map(); + const uid = this.threadLocal.get("uid"); + + do { + if (pdfMode) { + contentText = (snapshot.parsed?.content || snapshot.text || "").trim(); + break; + } + + if ( + snapshot.maxElemDepth! > 256 || + (!uid && snapshot.elemCount! > 10_000) || + snapshot.elemCount! > 80_000 + ) { + this.logger.warn("Degrading to text to protect the server", { + url: snapshot.href, + elemDepth: snapshot.maxElemDepth, + elemCount: snapshot.elemCount, + }); + contentText = (snapshot.text || "").trimEnd(); + break; + } + + const noGFMOpts = this.threadLocal.get("noGfm"); + const imageRetention = this.threadLocal.get( + "retainImages", + ) as CrawlerOptions["retainImages"]; + let imgIdx = 0; + const urlToAltMap: { [k: string]: string | undefined } = {}; + const customRules: { [k: string]: Rule } = { + "img-retention": { + filter: "img", + replacement: (_content: string, node: HTMLElement) => { + if (imageRetention === "none") { + return ""; } - Object.assign(f, { - screenshotUrl: snapshot.screenshotUrl, + const alt = cleanAttribute(node.getAttribute("alt")); + + if (imageRetention === "alt") { + return alt ? `(Image ${++imgIdx}: ${alt})` : ""; + } + const originalSrc = (node.getAttribute("src") || "").trim(); + let linkPreferredSrc = originalSrc; + const maybeSrcSet: string = ( + node.getAttribute("srcset") || "" + ).trim(); + if (!linkPreferredSrc && maybeSrcSet) { + linkPreferredSrc = maybeSrcSet + .split(",") + .map((x) => x.trim()) + .filter(Boolean)[0]; + } + if (!linkPreferredSrc || linkPreferredSrc.startsWith("data:")) { + const dataSrc = (node.getAttribute("data-src") || "").trim(); + if (dataSrc && !dataSrc.startsWith("data:")) { + linkPreferredSrc = dataSrc; + } + } + + let src; + try { + src = new URL( + linkPreferredSrc, + snapshot.rebase || nominalUrl, + ).toString(); + } catch (_err) { + void 0; + } + if (!src) { + return ""; + } + + const keySrc = ( + originalSrc.startsWith("data:") + ? this.dataUrlToBlobUrl(originalSrc, snapshot.rebase) + : src + ).trim(); + const mapped = urlToAltMap[keySrc]; + const imgSerial = ++imgIdx; + const idxArr = imageIdxTrack.has(keySrc) + ? imageIdxTrack.get(keySrc)! + : []; + idxArr.push(imgSerial); + imageIdxTrack.set(keySrc, idxArr); + + if (mapped) { + imageSummary[keySrc] = mapped || alt; + + if (imageRetention === "alt_p") { + return `(Image ${imgSerial}: ${mapped || alt})`; + } + + if (imgDataUrlToObjectUrl) { + return `![Image ${imgSerial}: ${mapped || alt}](${keySrc})`; + } + + return `![Image ${imgSerial}: ${mapped || alt}](${src})`; + } else if (imageRetention === "alt_p") { + return alt ? `(Image ${imgSerial}: ${alt})` : ""; + } + + imageSummary[keySrc] = alt || ""; + + if (imgDataUrlToObjectUrl) { + return alt + ? `![Image ${imgSerial}: ${alt}](${keySrc})` + : `![Image ${imgSerial}](${keySrc})`; + } + + return alt + ? `![Image ${imgSerial}: ${alt}](${src})` + : `![Image ${imgSerial}](${src})`; + }, + } as Rule, + }; + const optsMixin = { + url: snapshot.rebase || nominalUrl, + customRules, + customKeep: noGFMOpts === "table" ? "table" : undefined, + imgDataUrlToObjectUrl, + } as const; + + const jsDomElementOfHTML = this.jsdomControl.snippetToElement( + snapshot.html, + snapshot.href, + ); + let toBeTurnedToMd = jsDomElementOfHTML; + let turnDownService = this.getTurndown({ ...optsMixin }); + if (!mode.includes("markdown") && snapshot.parsed?.content) { + const jsDomElementOfParsed = this.jsdomControl.snippetToElement( + snapshot.parsed.content, + snapshot.href, + ); + const par1 = this.jsdomControl.runTurndown( + turnDownService, + jsDomElementOfHTML, + ); + imgIdx = 0; + const par2 = snapshot.parsed.content + ? this.jsdomControl.runTurndown(turnDownService, jsDomElementOfParsed) + : ""; + + // If Readability did its job + if (par2.length >= 0.3 * par1.length) { + turnDownService = this.getTurndown({ noRules: true, ...optsMixin }); + imgIdx = 0; + if (snapshot.parsed.content) { + toBeTurnedToMd = jsDomElementOfParsed; + } + } + } + + if (!noGFMOpts) { + turnDownService = turnDownService.use( + noGFMOpts === "table" ? this.gfmNoTable : this.gfmPlugin, + ); + } + + // _p is the special suffix for withGeneratedAlt + if (snapshot.imgs?.length && imageRetention?.endsWith("_p")) { + const tasks = _.uniqBy(snapshot.imgs || [], "src").map(async (x) => { + const r = await this.altTextService + .getAltText(x) + .catch((err: any) => { + this.logger.warn(`Failed to get alt text for ${x.src}`, { + err: marshalErrorLike(err), + }); + return undefined; }); + if (r && x.src) { + // note x.src here is already rebased to absolute url by browser/upstream. + const keySrc = ( + x.src.startsWith("data:") + ? this.dataUrlToBlobUrl(x.src, snapshot.rebase) + : x.src + ).trim(); + urlToAltMap[keySrc] = r; + } + }); - Object.defineProperty(f, 'textRepresentation', { value: `${f.screenshotUrl}\n`, enumerable: false, configurable: true }); - } - if (mode.includes('pageshot')) { - modeOK = true; - if (snapshot.pageshot && !snapshot.pageshotUrl) { - const fid = `instant-screenshots/${randomUUID()}`; - await this.firebaseObjectStorage.saveFile(fid, snapshot.pageshot, { - metadata: { - contentType: 'image/png', - } - }); - snapshot.pageshotUrl = await this.firebaseObjectStorage.signDownloadUrl(fid, Date.now() + urlValidMs); - } - Object.assign(f, { - html: snapshot.html, - pageshotUrl: snapshot.pageshotUrl, + await Promise.all(tasks); + } + + if (toBeTurnedToMd) { + try { + contentText = this.jsdomControl + .runTurndown(turnDownService, toBeTurnedToMd) + .trim(); + imgIdx = 0; + } catch (err) { + this.logger.warn(`Turndown failed to run, retrying without plugins`, { + err, + }); + const vanillaTurnDownService = this.getTurndown({ ...optsMixin }); + try { + contentText = this.jsdomControl + .runTurndown(vanillaTurnDownService, toBeTurnedToMd) + .trim(); + imgIdx = 0; + } catch (err2) { + this.logger.warn(`Turndown failed to run, giving up`, { + err: err2, }); - Object.defineProperty(f, 'textRepresentation', { value: `${f.pageshotUrl}\n`, enumerable: false, configurable: true }); + } } - if (mode.includes('html')) { - modeOK = true; - Object.assign(f, { - html: snapshot.html, + } + + if ( + this.isPoorlyTransformed(contentText, toBeTurnedToMd) && + toBeTurnedToMd !== jsDomElementOfHTML + ) { + toBeTurnedToMd = jsDomElementOfHTML; + try { + contentText = this.jsdomControl + .runTurndown(turnDownService, jsDomElementOfHTML) + .trim(); + imgIdx = 0; + } catch (err) { + this.logger.warn(`Turndown failed to run, retrying without plugins`, { + err, + }); + const vanillaTurnDownService = this.getTurndown({ ...optsMixin }); + try { + contentText = this.jsdomControl + .runTurndown(vanillaTurnDownService, jsDomElementOfHTML) + .trim(); + imgIdx = 0; + } catch (err2) { + this.logger.warn(`Turndown failed to run, giving up`, { + err: err2, }); - - Object.defineProperty(f, 'textRepresentation', { value: snapshot.html, enumerable: false, configurable: true }); + } } + } + if ( + mode === "content" && + this.isPoorlyTransformed(contentText, toBeTurnedToMd) + ) { + contentText = (snapshot.text || "").trimEnd(); + } + } while (false); - let pdfMode = false; - // in case of Google Web Cache content - if (snapshot.pdfs?.length && (!snapshot.title || snapshot.title.startsWith('cache:'))) { - const pdf = await this.pdfExtractor.cachedExtract(snapshot.pdfs[0], - this.threadLocal.get('cacheTolerance'), - snapshot.pdfs[0].startsWith('http') ? undefined : snapshot.href, - ); - if (pdf) { - pdfMode = true; - snapshot.title = pdf.meta?.Title; - snapshot.text = pdf.text || snapshot.text; - snapshot.parsed = { - content: pdf.content, - textContent: pdf.content, - length: pdf.content?.length, - byline: pdf.meta?.Author, - lang: pdf.meta?.Language || undefined, - title: pdf.meta?.Title, - publishedTime: this.pdfExtractor.parsePdfDate(pdf.meta?.ModDate || pdf.meta?.CreationDate)?.toISOString(), - }; - } + const formatted: FormattedPage = { + title: (snapshot.parsed?.title || snapshot.title || "").trim(), + description: (snapshot.description || "").trim(), + url: nominalUrl?.toString() || snapshot.href?.trim(), + content: contentText, + publishedTime: snapshot.parsed?.publishedTime || undefined, + }; + + if (snapshot.status) { + const code = snapshot.status; + const n = code - 200; + if (n < 0 || n >= 200) { + const text = snapshot.statusText || STATUS_CODES[code]; + formatted.warning ??= ""; + const msg = `Target URL returned error ${code}${text ? `: ${text}` : ""}`; + formatted.warning = `${formatted.warning}${formatted.warning ? "\n" : ""}${msg}`; + } + } + + if (this.threadLocal.get("withImagesSummary")) { + formatted.images = _(imageSummary) + .toPairs() + .map(([url, alt], i) => { + const idxTrack = imageIdxTrack.get(url); + const tag = idxTrack?.length + ? `Image ${_.uniq(idxTrack).join(",")}` + : `Hidden Image ${i + 1}`; + + return [`${tag}${alt ? `: ${alt}` : ""}`, url]; + }) + .fromPairs() + .value(); + } + if (this.threadLocal.get("withLinksSummary")) { + const links = (await this.jsdomControl.inferSnapshot(snapshot)).links; + + if (this.threadLocal.get("withLinksSummary") === "all") { + formatted.links = links; + } else { + formatted.links = _(links) + .filter(([_label, href]) => !hasDisallowedSummaryLinkScheme(href)) + .uniqBy(1) + .fromPairs() + .value(); + } + } + + if (countGPTToken(formatted.content) < 200) { + formatted.warning ??= ""; + if (snapshot.isIntermediate) { + const msg = + "This page maybe not yet fully loaded, consider explicitly specify a timeout."; + formatted.warning = `${formatted.warning}${formatted.warning ? "\n" : ""}${msg}`; + } + if (snapshot.childFrames?.length && !this.threadLocal.get("withIframe")) { + const msg = + "This page contains iframe that are currently hidden, consider enabling iframe processing."; + formatted.warning = `${formatted.warning}${formatted.warning ? "\n" : ""}${msg}`; + } + if (snapshot.shadowExpanded && !this.threadLocal.get("withShadowDom")) { + const msg = + "This page contains shadow DOM that are currently hidden, consider enabling shadow DOM processing."; + formatted.warning = `${formatted.warning}${formatted.warning ? "\n" : ""}${msg}`; + } + if ( + snapshot.html.includes("captcha") || + snapshot.html.includes("cf-turnstile-response") + ) { + const msg = + "This page maybe requiring CAPTCHA, please make sure you are authorized to access this page."; + formatted.warning = `${formatted.warning}${formatted.warning ? "\n" : ""}${msg}`; + } + if (snapshot.isFromCache) { + const msg = + "This is a cached snapshot of the original page, consider retry with caching opt-out."; + formatted.warning = `${formatted.warning}${formatted.warning ? "\n" : ""}${msg}`; + } + } + + Object.assign(f, formatted); + + const textRepresentation = function (this: typeof formatted) { + const mixins = []; + if (this.publishedTime) { + mixins.push(`Published Time: ${this.publishedTime}`); + } + const suffixMixins = []; + if (this.images) { + const imageSummaryChunks = ["Images:"]; + for (const [k, v] of Object.entries(this.images)) { + imageSummaryChunks.push(`- ![${k}](${v})`); } - - if (mode.includes('text')) { - modeOK = true; - Object.assign(f, { - text: snapshot.text, - }); - Object.defineProperty(f, 'textRepresentation', { value: snapshot.text, enumerable: false, configurable: true }); + if (imageSummaryChunks.length === 1) { + imageSummaryChunks.push( + "This page does not seem to contain any images.", + ); } - - if (mode.includes('lm')) { - modeOK = true; - f.content = snapshot.parsed?.textContent; + suffixMixins.push(imageSummaryChunks.join("\n")); + } + if (this.links) { + const linkSummaryChunks = ["Links/Buttons:"]; + if (Array.isArray(this.links)) { + for (const [k, v] of this.links) { + linkSummaryChunks.push(`- [${k}](${v})`); + } + } else { + for (const [k, v] of Object.entries(this.links)) { + linkSummaryChunks.push(`- [${k}](${v})`); + } } - - if (modeOK && (mode.includes('lm') || - (!mode.includes('markdown') && !mode.includes('content'))) - ) { - const dt = Date.now() - t0; - this.logger.debug(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt }); - - const formatted: FormattedPage = { - title: (snapshot.parsed?.title || snapshot.title || '').trim(), - description: (snapshot.description || '').trim(), - url: nominalUrl?.toString() || snapshot.href?.trim(), - publishedTime: snapshot.parsed?.publishedTime || undefined, - }; - - Object.assign(f, formatted); - - return f; + if (linkSummaryChunks.length === 1) { + linkSummaryChunks.push( + "This page does not seem to contain any buttons/links.", + ); } + suffixMixins.push(linkSummaryChunks.join("\n")); + } - const imgDataUrlToObjectUrl = !Boolean(this.threadLocal.get('keepImgDataUrl')); + if (this.warning) { + mixins.push( + this.warning + .split("\n") + .map((v) => `Warning: ${v}`) + .join("\n"), + ); + } - let contentText = ''; - const imageSummary = {} as { [k: string]: string; }; - const imageIdxTrack = new Map(); - const uid = this.threadLocal.get('uid'); + if (mode.includes("markdown")) { + return `${mixins.length ? `${mixins.join("\n\n")}\n\n` : ""}${this.content} +${suffixMixins.length ? `\n${suffixMixins.join("\n\n")}\n` : ""}`; + } - do { - if (pdfMode) { - contentText = (snapshot.parsed?.content || snapshot.text || '').trim(); - break; - } - - if ( - snapshot.maxElemDepth! > 256 || - (!uid && snapshot.elemCount! > 10_000) || - snapshot.elemCount! > 80_000 - ) { - this.logger.warn('Degrading to text to protect the server', { url: snapshot.href, elemDepth: snapshot.maxElemDepth, elemCount: snapshot.elemCount }); - contentText = (snapshot.text || '').trimEnd(); - break; - } - - const noGFMOpts = this.threadLocal.get('noGfm'); - const imageRetention = this.threadLocal.get('retainImages') as CrawlerOptions['retainImages']; - let imgIdx = 0; - const urlToAltMap: { [k: string]: string | undefined; } = {}; - const customRules: { [k: string]: Rule; } = { - 'img-retention': { - filter: 'img', - replacement: (_content: string, node: HTMLElement) => { - if (imageRetention === 'none') { - return ''; - } - const alt = cleanAttribute(node.getAttribute('alt')); - - if (imageRetention === 'alt') { - return alt ? `(Image ${++imgIdx}: ${alt})` : ''; - } - const originalSrc = (node.getAttribute('src') || '').trim(); - let linkPreferredSrc = originalSrc; - const maybeSrcSet: string = (node.getAttribute('srcset') || '').trim(); - if (!linkPreferredSrc && maybeSrcSet) { - linkPreferredSrc = maybeSrcSet.split(',').map((x) => x.trim()).filter(Boolean)[0]; - } - if (!linkPreferredSrc || linkPreferredSrc.startsWith('data:')) { - const dataSrc = (node.getAttribute('data-src') || '').trim(); - if (dataSrc && !dataSrc.startsWith('data:')) { - linkPreferredSrc = dataSrc; - } - } - - let src; - try { - src = new URL(linkPreferredSrc, snapshot.rebase || nominalUrl).toString(); - } catch (_err) { - void 0; - } - if (!src) { - return ''; - } - - const keySrc = (originalSrc.startsWith('data:') ? this.dataUrlToBlobUrl(originalSrc, snapshot.rebase) : src).trim(); - const mapped = urlToAltMap[keySrc]; - const imgSerial = ++imgIdx; - const idxArr = imageIdxTrack.has(keySrc) ? imageIdxTrack.get(keySrc)! : []; - idxArr.push(imgSerial); - imageIdxTrack.set(keySrc, idxArr); - - if (mapped) { - imageSummary[keySrc] = mapped || alt; - - if (imageRetention === 'alt_p') { - return `(Image ${imgSerial}: ${mapped || alt})`; - } - - if (imgDataUrlToObjectUrl) { - return `![Image ${imgSerial}: ${mapped || alt}](${keySrc})`; - } - - return `![Image ${imgSerial}: ${mapped || alt}](${src})`; - } else if (imageRetention === 'alt_p') { - return alt ? `(Image ${imgSerial}: ${alt})` : ''; - } - - imageSummary[keySrc] = alt || ''; - - if (imgDataUrlToObjectUrl) { - return alt ? `![Image ${imgSerial}: ${alt}](${keySrc})` : `![Image ${imgSerial}](${keySrc})`; - } - - return alt ? `![Image ${imgSerial}: ${alt}](${src})` : `![Image ${imgSerial}](${src})`; - } - } as Rule - }; - const optsMixin = { - url: snapshot.rebase || nominalUrl, - customRules, - customKeep: noGFMOpts === 'table' ? 'table' : undefined, - imgDataUrlToObjectUrl, - } as const; - - const jsDomElementOfHTML = this.jsdomControl.snippetToElement(snapshot.html, snapshot.href); - let toBeTurnedToMd = jsDomElementOfHTML; - let turnDownService = this.getTurndown({ ...optsMixin }); - if (!mode.includes('markdown') && snapshot.parsed?.content) { - const jsDomElementOfParsed = this.jsdomControl.snippetToElement(snapshot.parsed.content, snapshot.href); - const par1 = this.jsdomControl.runTurndown(turnDownService, jsDomElementOfHTML); - imgIdx = 0; - const par2 = snapshot.parsed.content ? this.jsdomControl.runTurndown(turnDownService, jsDomElementOfParsed) : ''; - - // If Readability did its job - if (par2.length >= 0.3 * par1.length) { - turnDownService = this.getTurndown({ noRules: true, ...optsMixin }); - imgIdx = 0; - if (snapshot.parsed.content) { - toBeTurnedToMd = jsDomElementOfParsed; - } - } - } - - if (!noGFMOpts) { - turnDownService = turnDownService.use(noGFMOpts === 'table' ? this.gfmNoTable : this.gfmPlugin); - } - - // _p is the special suffix for withGeneratedAlt - if (snapshot.imgs?.length && imageRetention?.endsWith('_p')) { - const tasks = _.uniqBy((snapshot.imgs || []), 'src').map(async (x) => { - const r = await this.altTextService.getAltText(x).catch((err: any) => { - this.logger.warn(`Failed to get alt text for ${x.src}`, { err: marshalErrorLike(err) }); - return undefined; - }); - if (r && x.src) { - // note x.src here is already rebased to absolute url by browser/upstream. - const keySrc = (x.src.startsWith('data:') ? this.dataUrlToBlobUrl(x.src, snapshot.rebase) : x.src).trim(); - urlToAltMap[keySrc] = r; - } - }); - - await Promise.all(tasks); - } - - if (toBeTurnedToMd) { - try { - contentText = this.jsdomControl.runTurndown(turnDownService, toBeTurnedToMd).trim(); - imgIdx = 0; - } catch (err) { - this.logger.warn(`Turndown failed to run, retrying without plugins`, { err }); - const vanillaTurnDownService = this.getTurndown({ ...optsMixin }); - try { - contentText = this.jsdomControl.runTurndown(vanillaTurnDownService, toBeTurnedToMd).trim(); - imgIdx = 0; - } catch (err2) { - this.logger.warn(`Turndown failed to run, giving up`, { err: err2 }); - } - } - } - - if ( - this.isPoorlyTransformed(contentText, toBeTurnedToMd) - && toBeTurnedToMd !== jsDomElementOfHTML - ) { - toBeTurnedToMd = jsDomElementOfHTML; - try { - contentText = this.jsdomControl.runTurndown(turnDownService, jsDomElementOfHTML).trim(); - imgIdx = 0; - } catch (err) { - this.logger.warn(`Turndown failed to run, retrying without plugins`, { err }); - const vanillaTurnDownService = this.getTurndown({ ...optsMixin }); - try { - contentText = this.jsdomControl.runTurndown(vanillaTurnDownService, jsDomElementOfHTML).trim(); - imgIdx = 0; - } catch (err2) { - this.logger.warn(`Turndown failed to run, giving up`, { err: err2 }); - } - } - } - if (mode === 'content' && this.isPoorlyTransformed(contentText, toBeTurnedToMd)) { - contentText = (snapshot.text || '').trimEnd(); - } - } while (false); - - const formatted: FormattedPage = { - title: (snapshot.parsed?.title || snapshot.title || '').trim(), - description: (snapshot.description || '').trim(), - url: nominalUrl?.toString() || snapshot.href?.trim(), - content: contentText, - publishedTime: snapshot.parsed?.publishedTime || undefined, - }; - - if (snapshot.status) { - const code = snapshot.status; - const n = code - 200; - if (n < 0 || n >= 200) { - const text = snapshot.statusText || STATUS_CODES[code]; - formatted.warning ??= ''; - const msg = `Target URL returned error ${code}${text ? `: ${text}` : ''}`; - formatted.warning = `${formatted.warning}${formatted.warning ? '\n' : ''}${msg}`; - } - } - - if (this.threadLocal.get('withImagesSummary')) { - formatted.images = - _(imageSummary) - .toPairs() - .map( - ([url, alt], i) => { - const idxTrack = imageIdxTrack.get(url); - const tag = idxTrack?.length ? `Image ${_.uniq(idxTrack).join(',')}` : `Hidden Image ${i + 1}`; - - return [`${tag}${alt ? `: ${alt}` : ''}`, url]; - } - ).fromPairs() - .value(); - } - if (this.threadLocal.get('withLinksSummary')) { - const links = (await this.jsdomControl.inferSnapshot(snapshot)).links; - - if (this.threadLocal.get('withLinksSummary') === 'all') { - formatted.links = links; - } else { - formatted.links = _(links).filter(([_label, href]) => !hasDisallowedSummaryLinkScheme(href)).uniqBy(1).fromPairs().value(); - } - } - - if (countGPTToken(formatted.content) < 200) { - formatted.warning ??= ''; - if (snapshot.isIntermediate) { - const msg = 'This page maybe not yet fully loaded, consider explicitly specify a timeout.'; - formatted.warning = `${formatted.warning}${formatted.warning ? '\n' : ''}${msg}`; - } - if (snapshot.childFrames?.length && !this.threadLocal.get('withIframe')) { - const msg = 'This page contains iframe that are currently hidden, consider enabling iframe processing.'; - formatted.warning = `${formatted.warning}${formatted.warning ? '\n' : ''}${msg}`; - } - if (snapshot.shadowExpanded && !this.threadLocal.get('withShadowDom')) { - const msg = 'This page contains shadow DOM that are currently hidden, consider enabling shadow DOM processing.'; - formatted.warning = `${formatted.warning}${formatted.warning ? '\n' : ''}${msg}`; - } - if (snapshot.html.includes('captcha') || snapshot.html.includes('cf-turnstile-response')) { - const msg = 'This page maybe requiring CAPTCHA, please make sure you are authorized to access this page.'; - formatted.warning = `${formatted.warning}${formatted.warning ? '\n' : ''}${msg}`; - } - if (snapshot.isFromCache) { - const msg = 'This is a cached snapshot of the original page, consider retry with caching opt-out.'; - formatted.warning = `${formatted.warning}${formatted.warning ? '\n' : ''}${msg}`; - } - } - - Object.assign(f, formatted); - - const textRepresentation = (function (this: typeof formatted) { - const mixins = []; - if (this.publishedTime) { - mixins.push(`Published Time: ${this.publishedTime}`); - } - const suffixMixins = []; - if (this.images) { - const imageSummaryChunks = ['Images:']; - for (const [k, v] of Object.entries(this.images)) { - imageSummaryChunks.push(`- ![${k}](${v})`); - } - if (imageSummaryChunks.length === 1) { - imageSummaryChunks.push('This page does not seem to contain any images.'); - } - suffixMixins.push(imageSummaryChunks.join('\n')); - } - if (this.links) { - const linkSummaryChunks = ['Links/Buttons:']; - if (Array.isArray(this.links)) { - for (const [k, v] of this.links) { - linkSummaryChunks.push(`- [${k}](${v})`); - } - } else { - for (const [k, v] of Object.entries(this.links)) { - linkSummaryChunks.push(`- [${k}](${v})`); - } - } - if (linkSummaryChunks.length === 1) { - linkSummaryChunks.push('This page does not seem to contain any buttons/links.'); - } - suffixMixins.push(linkSummaryChunks.join('\n')); - } - - if (this.warning) { - mixins.push(this.warning.split('\n').map((v) => `Warning: ${v}`).join('\n')); - } - - if (mode.includes('markdown')) { - return `${mixins.length ? `${mixins.join('\n\n')}\n\n` : ''}${this.content} -${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`; - } - - return `Title: ${this.title} + return `Title: ${this.title} URL Source: ${this.url} -${mixins.length ? `\n${mixins.join('\n\n')}\n` : ''} +${mixins.length ? `\n${mixins.join("\n\n")}\n` : ""} Markdown Content: ${this.content} -${suffixMixins.length ? `\n${suffixMixins.join('\n\n')}\n` : ''}`; - }).call(formatted); +${suffixMixins.length ? `\n${suffixMixins.join("\n\n")}\n` : ""}`; + }.call(formatted); - Object.defineProperty(f, 'textRepresentation', { value: textRepresentation, enumerable: false }); + Object.defineProperty(f, "textRepresentation", { + value: textRepresentation, + enumerable: false, + }); - const dt = Date.now() - t0; - this.logger.debug(`Formatting took ${dt}ms`, { mode, url: nominalUrl?.toString(), dt }); + const dt = Date.now() - t0; + this.logger.debug(`Formatting took ${dt}ms`, { + mode, + url: nominalUrl?.toString(), + dt, + }); - return f as FormattedPage; + return f as FormattedPage; + } + + dataUrlToBlobUrl(dataUrl: string, baseUrl: string = "http://localhost/") { + const refUrl = new URL(baseUrl); + const mappedUrl = new URL( + `blob:${refUrl.origin || "localhost"}/${md5Hasher.hash(dataUrl)}`, + ); + + return mappedUrl.href; + } + + async getGeneralSnapshotMixins(snapshot: PageSnapshot) { + let inferred; + const mixin: any = {}; + if (this.threadLocal.get("withImagesSummary")) { + inferred ??= await this.jsdomControl.inferSnapshot(snapshot); + const imageSummary = {} as { [k: string]: string }; + const imageIdxTrack = new Map(); + + let imgIdx = 0; + + for (const img of inferred.imgs) { + const imgSerial = ++imgIdx; + const keySrc = ( + img.src.startsWith("data:") + ? this.dataUrlToBlobUrl(img.src, snapshot.rebase) + : img.src + ).trim(); + const idxArr = imageIdxTrack.has(keySrc) + ? imageIdxTrack.get(keySrc)! + : []; + idxArr.push(imgSerial); + imageIdxTrack.set(keySrc, idxArr); + imageSummary[keySrc] = img.alt || ""; + } + + mixin.images = _(imageSummary) + .toPairs() + .map(([url, alt], i) => { + const idxTrack = imageIdxTrack.get(url); + const tag = idxTrack?.length + ? `Image ${_.uniq(idxTrack).join(",")}` + : `Hidden Image ${i + 1}`; + + return [`${tag}${alt ? `: ${alt}` : ""}`, url]; + }) + .fromPairs() + .value(); + } + if (this.threadLocal.get("withLinksSummary")) { + inferred ??= await this.jsdomControl.inferSnapshot(snapshot); + if (this.threadLocal.get("withLinksSummary") === "all") { + mixin.links = inferred.links; + } else { + mixin.links = _(inferred.links) + .filter(([_label, href]) => !hasDisallowedSummaryLinkScheme(href)) + .uniqBy(1) + .fromPairs() + .value(); + } + } + if (snapshot.status) { + const code = snapshot.status; + const n = code - 200; + if (n < 0 || n >= 200) { + const text = snapshot.statusText || STATUS_CODES[code]; + mixin.warning ??= ""; + const msg = `Target URL returned error ${code}${text ? `: ${text}` : ""}`; + mixin.warning = `${mixin.warning}${mixin.warning ? "\n" : ""}${msg}`; + } } - dataUrlToBlobUrl(dataUrl: string, baseUrl: string = 'http://localhost/') { - const refUrl = new URL(baseUrl); - const mappedUrl = new URL(`blob:${refUrl.origin || 'localhost'}/${md5Hasher.hash(dataUrl)}`); + return mixin; + } - return mappedUrl.href; + getTurndown(options?: { + noRules?: boolean | string; + url?: string | URL; + imgDataUrlToObjectUrl?: boolean; + removeImages?: boolean | "src"; + customRules?: { [k: string]: Rule }; + customKeep?: Filter; + }) { + const turndownOpts = this.threadLocal.get("turndownOpts"); + const turnDownService = new TurndownService({ + ...turndownOpts, + codeBlockStyle: "fenced", + preformattedCode: true, + } as any); + if (options?.customKeep) { + turnDownService.keep(options.customKeep); + } + if (!options?.noRules) { + turnDownService.addRule("remove-irrelevant", { + filter: [ + "meta", + "style", + "script", + "noscript", + "link", + "textarea", + "select", + ], + replacement: () => "", + }); + turnDownService.addRule("truncate-svg", { + filter: "svg" as any, + replacement: () => "", + }); + turnDownService.addRule("title-as-h1", { + filter: ["title"], + replacement: (innerText) => `${innerText}\n===============\n`, + }); } - async getGeneralSnapshotMixins(snapshot: PageSnapshot) { - let inferred; - const mixin: any = {}; - if (this.threadLocal.get('withImagesSummary')) { - inferred ??= await this.jsdomControl.inferSnapshot(snapshot); - const imageSummary = {} as { [k: string]: string; }; - const imageIdxTrack = new Map(); + if (options?.imgDataUrlToObjectUrl) { + turnDownService.addRule("data-url-to-pseudo-object-url", { + filter: (node) => + Boolean( + node.tagName === "IMG" && + node.getAttribute("src")?.startsWith("data:"), + ), + replacement: (_content, node: any) => { + const src = (node.getAttribute("src") || "").trim(); + const alt = cleanAttribute(node.getAttribute("alt")) || ""; - let imgIdx = 0; + const blobUrl = this.dataUrlToBlobUrl(src, options.url?.toString()); - for (const img of inferred.imgs) { - const imgSerial = ++imgIdx; - const keySrc = (img.src.startsWith('data:') ? this.dataUrlToBlobUrl(img.src, snapshot.rebase) : img.src).trim(); - const idxArr = imageIdxTrack.has(keySrc) ? imageIdxTrack.get(keySrc)! : []; - idxArr.push(imgSerial); - imageIdxTrack.set(keySrc, idxArr); - imageSummary[keySrc] = img.alt || ''; - } - - mixin.images = - _(imageSummary) - .toPairs() - .map( - ([url, alt], i) => { - const idxTrack = imageIdxTrack.get(url); - const tag = idxTrack?.length ? `Image ${_.uniq(idxTrack).join(',')}` : `Hidden Image ${i + 1}`; - - return [`${tag}${alt ? `: ${alt}` : ''}`, url]; - } - ).fromPairs() - .value(); - } - if (this.threadLocal.get('withLinksSummary')) { - inferred ??= await this.jsdomControl.inferSnapshot(snapshot); - if (this.threadLocal.get('withLinksSummary') === 'all') { - mixin.links = inferred.links; - } else { - mixin.links = _(inferred.links).filter(([_label, href]) => !hasDisallowedSummaryLinkScheme(href)).uniqBy(1).fromPairs().value(); - } - } - if (snapshot.status) { - const code = snapshot.status; - const n = code - 200; - if (n < 0 || n >= 200) { - const text = snapshot.statusText || STATUS_CODES[code]; - mixin.warning ??= ''; - const msg = `Target URL returned error ${code}${text ? `: ${text}` : ''}`; - mixin.warning = `${mixin.warning}${mixin.warning ? '\n' : ''}${msg}`; - } - } - - return mixin; + return `![${alt}](${blobUrl})`; + }, + }); } - getTurndown(options?: { - noRules?: boolean | string, - url?: string | URL; - imgDataUrlToObjectUrl?: boolean; - removeImages?: boolean | 'src'; - customRules?: { [k: string]: Rule; }; - customKeep?: Filter; - }) { - const turndownOpts = this.threadLocal.get('turndownOpts'); - const turnDownService = new TurndownService({ - ...turndownOpts, - codeBlockStyle: 'fenced', - preformattedCode: true, - } as any); - if (options?.customKeep) { - turnDownService.keep(options.customKeep); - } - if (!options?.noRules) { - turnDownService.addRule('remove-irrelevant', { - filter: ['meta', 'style', 'script', 'noscript', 'link', 'textarea', 'select'], - replacement: () => '' - }); - turnDownService.addRule('truncate-svg', { - filter: 'svg' as any, - replacement: () => '' - }); - turnDownService.addRule('title-as-h1', { - filter: ['title'], - replacement: (innerText) => `${innerText}\n===============\n` - }); - } - - if (options?.imgDataUrlToObjectUrl) { - turnDownService.addRule('data-url-to-pseudo-object-url', { - filter: (node) => Boolean(node.tagName === 'IMG' && node.getAttribute('src')?.startsWith('data:')), - replacement: (_content, node: any) => { - const src = (node.getAttribute('src') || '').trim(); - const alt = cleanAttribute(node.getAttribute('alt')) || ''; - - const blobUrl = this.dataUrlToBlobUrl(src, options.url?.toString()); - - return `![${alt}](${blobUrl})`; - } - }); - } - - if (options?.customRules) { - for (const [k, v] of Object.entries(options.customRules)) { - turnDownService.addRule(k, v); - } - } - - turnDownService.addRule('improved-heading', { - filter: ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'], - replacement: (content, node, options) => { - const hLevel = Number(node.nodeName.charAt(1)); - if (options.headingStyle === 'setext' && hLevel < 3) { - const underline = _.repeat((hLevel === 1 ? '=' : '-'), Math.min(128, content.length)); - return ( - '\n\n' + content + '\n' + underline + '\n\n' - ); - } else { - return '\n\n' + _.repeat('#', hLevel) + ' ' + content + '\n\n'; - } - } - }); - - turnDownService.addRule('improved-paragraph', { - filter: 'p', - replacement: (innerText) => { - const trimmed = innerText.trim(); - if (!trimmed) { - return ''; - } - - return `${trimmed.replace(/\n{3,}/g, '\n\n')}\n\n`; - } - }); - - let realLinkStyle: 'inlined' | 'collapsed' | 'shortcut' | 'referenced' | 'discarded' = 'inlined'; - if (turndownOpts?.linkStyle === 'referenced' || turndownOpts?.linkReferenceStyle) { - realLinkStyle = 'referenced'; - if (turndownOpts?.linkReferenceStyle === 'collapsed') { - realLinkStyle = 'collapsed'; - } else if (turndownOpts?.linkReferenceStyle === 'shortcut') { - realLinkStyle = 'shortcut'; - } else if (turndownOpts?.linkReferenceStyle === 'discarded') { - realLinkStyle = 'discarded'; - } - } else if (turndownOpts?.linkStyle === 'discarded') { - realLinkStyle = 'discarded'; - } - - turnDownService.addRule('improved-link', { - filter: function (node, _options) { - return Boolean( - node.nodeName === 'A' && - node.getAttribute('href') - ); - }, - - replacement: function (this: { references: string[]; }, content, node: any) { - var href = node.getAttribute('href'); - let title = cleanAttribute(node.getAttribute('title')); - if (title) title = ` "${escapeMarkdownTitleAttribute(title)}"`; - let replacement; - let reference; - const fixedContent = content.replace(/\s+/g, ' ').trim(); - let fixedHref = href; - if (options?.url) { - try { - fixedHref = new URL(fixedHref, options.url).toString(); - } catch (_err) { - void 0; - } - } - - switch (realLinkStyle) { - case 'inlined': - replacement = `[${fixedContent}](${fixedHref}${title || ''})`; - reference = undefined; - break; - case 'collapsed': - replacement = `[${fixedContent}][]`; - reference = `[${fixedContent}]: ${fixedHref}${title}`; - break; - case 'shortcut': - replacement = `[${fixedContent}]`; - reference = `[${fixedContent}]: ${fixedHref}${title}`; - break; - case 'discarded': - replacement = content; - reference = undefined; - break; - default: - const id = this.references.length + 1; - replacement = `[${fixedContent}][${id}]`; - reference = `[${id}]${fixedHref}${title}`; - } - - if (reference) { - this.references.push(reference); - } - - return replacement; - }, - - // @ts-ignore - references: [], - - append: function (this: { references: string[]; }) { - let references = ''; - if (this.references.length) { - references = `\n\n${this.references.join('\n')}\n\n`; - this.references = []; // Reset references - } - return references; - } - }); - turnDownService.addRule('improved-code', { - filter: function (node: any) { - let hasSiblings = node.previousSibling || node.nextSibling; - let isCodeBlock = node.parentNode.nodeName === 'PRE' && !hasSiblings; - - return node.nodeName === 'CODE' && !isCodeBlock; - }, - - replacement: function (inputContent: any) { - if (!inputContent) return ''; - let content = inputContent; - - let delimiter = '`'; - let matches = content.match(/`+/gm) || []; - while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + '`'; - if (content.includes('\n')) { - delimiter = '```'; - } - - let extraSpace = delimiter === '```' ? '\n' : /^`|^ .*?[^ ].* $|`$/.test(content) ? ' ' : ''; - - return delimiter + extraSpace + content + (delimiter === '```' && !content.endsWith(extraSpace) ? extraSpace : '') + delimiter; - } - }); - - return turnDownService; + if (options?.customRules) { + for (const [k, v] of Object.entries(options.customRules)) { + turnDownService.addRule(k, v); + } } + turnDownService.addRule("improved-heading", { + filter: ["h1", "h2", "h3", "h4", "h5", "h6"], + replacement: (content, node, options) => { + const hLevel = Number(node.nodeName.charAt(1)); + if (options.headingStyle === "setext" && hLevel < 3) { + const underline = _.repeat( + hLevel === 1 ? "=" : "-", + Math.min(128, content.length), + ); + return "\n\n" + content + "\n" + underline + "\n\n"; + } else { + return "\n\n" + _.repeat("#", hLevel) + " " + content + "\n\n"; + } + }, + }); - isPoorlyTransformed(content?: string, node?: Element) { - if (!content) { - return true; + turnDownService.addRule("improved-paragraph", { + filter: "p", + replacement: (innerText) => { + const trimmed = innerText.trim(); + if (!trimmed) { + return ""; } - if (content.startsWith('<') && content.endsWith('>')) { - return true; - } + return `${trimmed.replace(/\n{3,}/g, "\n\n")}\n\n`; + }, + }); - if (!this.threadLocal.get('noGfm') && content.includes('')) { - if (node?.textContent && content.length > node.textContent.length * 0.8) { - return true; - } - - const tableElms = node?.querySelectorAll('table') || []; - const deepTableElms = node?.querySelectorAll('table table'); - if (node && tableElms.length) { - const wrappingTables = _.without(tableElms, ...Array.from(deepTableElms || [])); - const tableTextsLength = _.sum(wrappingTables.map((x) => (x.innerHTML?.length || 0))); - - if (tableTextsLength / (content.length) > 0.6) { - return true; - } - } - - const tbodyElms = node?.querySelectorAll('tbody') || []; - const deepTbodyElms = node?.querySelectorAll('tbody tbody'); - if ((deepTbodyElms?.length || 0) / tbodyElms.length > 0.6) { - return true; - } - } - - return false; + let realLinkStyle: + | "inlined" + | "collapsed" + | "shortcut" + | "referenced" + | "discarded" = "inlined"; + if ( + turndownOpts?.linkStyle === "referenced" || + turndownOpts?.linkReferenceStyle + ) { + realLinkStyle = "referenced"; + if (turndownOpts?.linkReferenceStyle === "collapsed") { + realLinkStyle = "collapsed"; + } else if (turndownOpts?.linkReferenceStyle === "shortcut") { + realLinkStyle = "shortcut"; + } else if (turndownOpts?.linkReferenceStyle === "discarded") { + realLinkStyle = "discarded"; + } + } else if (turndownOpts?.linkStyle === "discarded") { + realLinkStyle = "discarded"; } - async createSnapshotFromFile(url: URL, file: FancyFile, overrideContentType?: string, overrideFileName?: string) { - if (overrideContentType === 'application/octet-stream') { - overrideContentType = undefined; + turnDownService.addRule("improved-link", { + filter: function (node, _options) { + return Boolean(node.nodeName === "A" && node.getAttribute("href")); + }, + + replacement: function ( + this: { references: string[] }, + content, + node: any, + ) { + var href = node.getAttribute("href"); + let title = cleanAttribute(node.getAttribute("title")); + if (title) title = ` "${escapeMarkdownTitleAttribute(title)}"`; + let replacement; + let reference; + const fixedContent = content.replace(/\s+/g, " ").trim(); + let fixedHref = href; + if (options?.url) { + try { + fixedHref = new URL(fixedHref, options.url).toString(); + } catch (_err) { + void 0; + } } - const contentType: string = (overrideContentType || await file.mimeType).toLowerCase(); - const fileName = overrideFileName || `${url.origin}${url.pathname}`; - const snapshot: PageSnapshot = { - title: '', - href: url.href, - html: '', - text: '' - }; - - if (contentType.startsWith('image/')) { - snapshot.html = `${fileName}`; - snapshot.title = fileName; - snapshot.imgs = [{ src: url.href }]; - - return snapshot; - } - try { - const encoding: string | undefined = contentType.includes('charset=') ? contentType.split('charset=')[1]?.trim().toLowerCase() : 'utf-8'; - if (contentType.startsWith('text/html')) { - if ((await file.size) > 1024 * 1024 * 32) { - throw new AssertionFailureError(`Failed to access ${url}: file too large`); - } - snapshot.html = await readFile(await file.filePath, encoding); - let innerCharset; - const peek = snapshot.html.slice(0, 1024); - innerCharset ??= peek.match(/]+text\/html;\s*?charset=([^>"]+)/i)?.[1]?.toLowerCase(); - innerCharset ??= peek.match(/]+charset="([^>"]+)"/i)?.[1]?.toLowerCase(); - if (innerCharset && innerCharset !== encoding) { - snapshot.html = await readFile(await file.filePath, innerCharset); - } - - return snapshot; - } - if (contentType.startsWith('text/') || contentType.startsWith('application/json')) { - if ((await file.size) > 1024 * 1024 * 32) { - throw new AssertionFailureError(`Failed to access ${url}: file too large`); - } - snapshot.text = await readFile(await file.filePath, encoding); - snapshot.html = `
${snapshot.text}
`; - - return snapshot; - } - if (contentType.startsWith('application/pdf')) { - snapshot.pdfs = [pathToFileURL(await file.filePath).href]; - - return snapshot; - } - } catch (err: any) { - this.logger.warn(`Failed to read from file: ${url}`, { err, url }); - throw new DataStreamBrokenError(`Failed to access ${url}: ${err?.message}`); + switch (realLinkStyle) { + case "inlined": + replacement = `[${fixedContent}](${fixedHref}${title || ""})`; + reference = undefined; + break; + case "collapsed": + replacement = `[${fixedContent}][]`; + reference = `[${fixedContent}]: ${fixedHref}${title}`; + break; + case "shortcut": + replacement = `[${fixedContent}]`; + reference = `[${fixedContent}]: ${fixedHref}${title}`; + break; + case "discarded": + replacement = content; + reference = undefined; + break; + default: + const id = this.references.length + 1; + replacement = `[${fixedContent}][${id}]`; + reference = `[${id}]${fixedHref}${title}`; } - throw new AssertionFailureError(`Failed to access ${url}: unexpected type ${contentType}`); + if (reference) { + this.references.push(reference); + } + + return replacement; + }, + + // @ts-ignore + references: [], + + append: function (this: { references: string[] }) { + let references = ""; + if (this.references.length) { + references = `\n\n${this.references.join("\n")}\n\n`; + this.references = []; // Reset references + } + return references; + }, + }); + turnDownService.addRule("improved-code", { + filter: function (node: any) { + let hasSiblings = node.previousSibling || node.nextSibling; + let isCodeBlock = node.parentNode.nodeName === "PRE" && !hasSiblings; + + return node.nodeName === "CODE" && !isCodeBlock; + }, + + replacement: function (inputContent: any) { + if (!inputContent) return ""; + let content = inputContent; + + let delimiter = "`"; + let matches = content.match(/`+/gm) || []; + while (matches.indexOf(delimiter) !== -1) delimiter = delimiter + "`"; + if (content.includes("\n")) { + delimiter = "```"; + } + + let extraSpace = + delimiter === "```" + ? "\n" + : /^`|^ .*?[^ ].* $|`$/.test(content) + ? " " + : ""; + + return ( + delimiter + + extraSpace + + content + + (delimiter === "```" && !content.endsWith(extraSpace) + ? extraSpace + : "") + + delimiter + ); + }, + }); + + return turnDownService; + } + + isPoorlyTransformed(content?: string, node?: Element) { + if (!content) { + return true; } + + if (content.startsWith("<") && content.endsWith(">")) { + return true; + } + + if ( + !this.threadLocal.get("noGfm") && + content.includes("") + ) { + if (node?.textContent && content.length > node.textContent.length * 0.8) { + return true; + } + + const tableElms = node?.querySelectorAll("table") || []; + const deepTableElms = node?.querySelectorAll("table table"); + if (node && tableElms.length) { + const wrappingTables = _.without( + tableElms, + ...Array.from(deepTableElms || []), + ); + const tableTextsLength = _.sum( + wrappingTables.map((x) => x.innerHTML?.length || 0), + ); + + if (tableTextsLength / content.length > 0.6) { + return true; + } + } + + const tbodyElms = node?.querySelectorAll("tbody") || []; + const deepTbodyElms = node?.querySelectorAll("tbody tbody"); + if ((deepTbodyElms?.length || 0) / tbodyElms.length > 0.6) { + return true; + } + } + + return false; + } + + async createSnapshotFromFile( + url: URL, + file: FancyFile, + overrideContentType?: string, + overrideFileName?: string, + ) { + if (overrideContentType === "application/octet-stream") { + overrideContentType = undefined; + } + + const contentType: string = ( + overrideContentType || (await file.mimeType) + ).toLowerCase(); + const fileName = overrideFileName || `${url.origin}${url.pathname}`; + const snapshot: PageSnapshot = { + title: "", + href: url.href, + html: "", + text: "", + }; + + if (contentType.startsWith("image/")) { + snapshot.html = `${fileName}`; + snapshot.title = fileName; + snapshot.imgs = [{ src: url.href }]; + + return snapshot; + } + try { + const encoding: string | undefined = contentType.includes("charset=") + ? contentType.split("charset=")[1]?.trim().toLowerCase() + : "utf-8"; + if (contentType.startsWith("text/html")) { + if ((await file.size) > 1024 * 1024 * 32) { + throw new AssertionFailureError( + `Failed to access ${url}: file too large`, + ); + } + snapshot.html = await readFile(await file.filePath, encoding); + let innerCharset; + const peek = snapshot.html.slice(0, 1024); + innerCharset ??= peek + .match(/]+text\/html;\s*?charset=([^>"]+)/i)?.[1] + ?.toLowerCase(); + innerCharset ??= peek + .match(/]+charset="([^>"]+)"/i)?.[1] + ?.toLowerCase(); + if (innerCharset && innerCharset !== encoding) { + snapshot.html = await readFile(await file.filePath, innerCharset); + } + + return snapshot; + } + if ( + contentType.startsWith("text/") || + contentType.startsWith("application/json") + ) { + if ((await file.size) > 1024 * 1024 * 32) { + throw new AssertionFailureError( + `Failed to access ${url}: file too large`, + ); + } + snapshot.text = await readFile(await file.filePath, encoding); + snapshot.html = `
${snapshot.text}
`; + + return snapshot; + } + if (contentType.startsWith("application/pdf")) { + snapshot.pdfs = [pathToFileURL(await file.filePath).href]; + + return snapshot; + } + } catch (err: any) { + this.logger.warn(`Failed to read from file: ${url}`, { err, url }); + throw new DataStreamBrokenError( + `Failed to access ${url}: ${err?.message}`, + ); + } + + throw new AssertionFailureError( + `Failed to access ${url}: unexpected type ${contentType}`, + ); + } } const snapshotFormatter = container.resolve(SnapshotFormatter); diff --git a/src/services/temp-file.ts b/src/services/temp-file.ts index 74f10ba..9699b74 100644 --- a/src/services/temp-file.ts +++ b/src/services/temp-file.ts @@ -1,23 +1,22 @@ -import { AbstractTempFileManger } from 'civkit/temp'; -import { rm } from 'fs/promises'; -import { singleton } from 'tsyringe'; -import { Finalizer } from './finalizer'; +import { AbstractTempFileManger } from "civkit/temp"; +import { rm } from "fs/promises"; +import { singleton } from "tsyringe"; +import { Finalizer } from "./finalizer"; @singleton() export class TempFileManager extends AbstractTempFileManger { + rootDir = ""; - rootDir = ''; + override async init() { + await this.dependencyReady(); + await super.init(); + this.emit("ready"); + } - override async init() { - await this.dependencyReady(); - await super.init(); - this.emit('ready'); - } + @Finalizer() + override async standDown() { + await super.standDown(); - @Finalizer() - override async standDown() { - await super.standDown(); - - await rm(this.rootDir, { recursive: true, force: true }); - } + await rm(this.rootDir, { recursive: true, force: true }); + } } diff --git a/src/services/threaded.ts b/src/services/threaded.ts index 9107ecd..1db5d96 100644 --- a/src/services/threaded.ts +++ b/src/services/threaded.ts @@ -1,66 +1,69 @@ -import 'reflect-metadata'; +import "reflect-metadata"; -import { singleton, container } from 'tsyringe'; -import { AbstractThreadedServiceRegistry } from 'civkit/threaded'; -import _ from 'lodash'; +import { singleton, container } from "tsyringe"; +import { AbstractThreadedServiceRegistry } from "civkit/threaded"; +import _ from "lodash"; -import { GlobalLogger } from './logger'; -import { AsyncLocalContext } from './async-context'; -import { PseudoTransfer } from './pseudo-transfer'; -import { cpus } from 'os'; -import { isMainThread } from 'worker_threads'; +import { GlobalLogger } from "./logger"; +import { AsyncLocalContext } from "./async-context"; +import { PseudoTransfer } from "./pseudo-transfer"; +import { cpus } from "os"; +import { isMainThread } from "worker_threads"; @singleton() export class ThreadedServiceRegistry extends AbstractThreadedServiceRegistry { - container = container; + container = container; - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - constructor( - protected globalLogger: GlobalLogger, - public asyncContext: AsyncLocalContext, - public pseudoTransfer: PseudoTransfer, - ) { - super(...arguments); - } - - setMaxWorkersByCpu() { - const cpuStat = cpus(); - - const evenCpuCycles = cpuStat.filter((_cpu, i) => i % 2 === 0).reduce((acc, cpu) => acc + cpu.times.user + cpu.times.sys, 0); - const oddCpuCycles = cpuStat.filter((_cpu, i) => i % 2 === 1).reduce((acc, cpu) => acc + cpu.times.user + cpu.times.sys, 0); - - const isLikelyHyperThreaded = (oddCpuCycles / evenCpuCycles) < 0.5; - - this.maxWorkers = isLikelyHyperThreaded ? cpuStat.length / 2 : cpuStat.length; - } - - override async init() { - await this.dependencyReady(); - await super.init(); - - if (isMainThread) { - this.setMaxWorkersByCpu(); - await Promise.all( - _.range(0, 2).map( - (_n) => - new Promise( - (resolve, reject) => { - this.createWorker() - .once('message', resolve) - .once('error', reject); - } - ) - ) - ); - } - - this.emit('ready'); + constructor( + protected globalLogger: GlobalLogger, + public asyncContext: AsyncLocalContext, + public pseudoTransfer: PseudoTransfer, + ) { + super(...arguments); + } + + setMaxWorkersByCpu() { + const cpuStat = cpus(); + + const evenCpuCycles = cpuStat + .filter((_cpu, i) => i % 2 === 0) + .reduce((acc, cpu) => acc + cpu.times.user + cpu.times.sys, 0); + const oddCpuCycles = cpuStat + .filter((_cpu, i) => i % 2 === 1) + .reduce((acc, cpu) => acc + cpu.times.user + cpu.times.sys, 0); + + const isLikelyHyperThreaded = oddCpuCycles / evenCpuCycles < 0.5; + + this.maxWorkers = isLikelyHyperThreaded + ? cpuStat.length / 2 + : cpuStat.length; + } + + override async init() { + await this.dependencyReady(); + await super.init(); + + if (isMainThread) { + this.setMaxWorkersByCpu(); + await Promise.all( + _.range(0, 2).map( + (_n) => + new Promise((resolve, reject) => { + this.createWorker() + .once("message", resolve) + .once("error", reject); + }), + ), + ); } + this.emit("ready"); + } } - const instance = container.resolve(ThreadedServiceRegistry); export default instance; -export const { Method, Param, Ctx, RPCReflect, Threaded } = instance.decorators(); +export const { Method, Param, Ctx, RPCReflect, Threaded } = + instance.decorators(); diff --git a/src/shared/3rd-party/brave-search.ts b/src/shared/3rd-party/brave-search.ts index d6fa834..eb8db43 100644 --- a/src/shared/3rd-party/brave-search.ts +++ b/src/shared/3rd-party/brave-search.ts @@ -1,48 +1,50 @@ export type WebSearchQueryParams = { - q: string; - count?: number; - offset?: number; - country?: string; - search_lang?: string; - [k: string]: any; + q: string; + count?: number; + offset?: number; + country?: string; + search_lang?: string; + [k: string]: any; }; export class BraveSearchHTTP { - constructor( - protected apiKey: string, - ) { } + constructor(protected apiKey: string) {} - async webSearch(query: WebSearchQueryParams, options?: { headers?: Record }) { - if (!this.apiKey) { - throw new Error('BRAVE_SEARCH_API_KEY is required for Brave search.'); - } - - const url = new URL('https://api.search.brave.com/res/v1/web/search'); - for (const [key, value] of Object.entries(query)) { - if (value === undefined || value === null) { - continue; - } - url.searchParams.set(key, `${value}`); - } - - const response = await fetch(url, { - headers: { - Accept: 'application/json', - 'X-Subscription-Token': this.apiKey, - ...(options?.headers || {}), - }, - }); - - if (!response.ok) { - const text = await response.text(); - const err: any = new Error(`Brave search failed: ${response.status} ${response.statusText} ${text}`.trim()); - err.status = response.status; - throw err; - } - - return { - parsed: await response.json(), - }; + async webSearch( + query: WebSearchQueryParams, + options?: { headers?: Record }, + ) { + if (!this.apiKey) { + throw new Error("BRAVE_SEARCH_API_KEY is required for Brave search."); } -} + const url = new URL("https://api.search.brave.com/res/v1/web/search"); + for (const [key, value] of Object.entries(query)) { + if (value === undefined || value === null) { + continue; + } + url.searchParams.set(key, `${value}`); + } + + const response = await fetch(url, { + headers: { + Accept: "application/json", + "X-Subscription-Token": this.apiKey, + ...(options?.headers || {}), + }, + }); + + if (!response.ok) { + const text = await response.text(); + const err: any = new Error( + `Brave search failed: ${response.status} ${response.statusText} ${text}`.trim(), + ); + err.status = response.status; + throw err; + } + + return { + parsed: await response.json(), + }; + } +} diff --git a/src/shared/3rd-party/brave-types.ts b/src/shared/3rd-party/brave-types.ts index 605d03d..23f0ee9 100644 --- a/src/shared/3rd-party/brave-types.ts +++ b/src/shared/3rd-party/brave-types.ts @@ -1,2 +1 @@ export type WebSearchOptionalHeaderOptions = Record; - diff --git a/src/shared/3rd-party/cloud-flare.ts b/src/shared/3rd-party/cloud-flare.ts index 2f1f87b..be99e09 100644 --- a/src/shared/3rd-party/cloud-flare.ts +++ b/src/shared/3rd-party/cloud-flare.ts @@ -1,32 +1,39 @@ export class CloudFlareHTTP { - constructor( - protected accountId?: string, - protected apiKey?: string, - ) { } + constructor( + protected accountId?: string, + protected apiKey?: string, + ) {} - async fetchBrowserRenderedHTML(input: { url: string }) { - if (!(this.accountId && this.apiKey)) { - throw new Error('CLOUD_FLARE_API_KEY must be configured as ACCOUNT_ID:API_KEY for browser rendering.'); - } - - const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${this.accountId}/browser-rendering/fetch`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${this.apiKey}`, - }, - body: JSON.stringify(input), - }); - - if (!response.ok) { - const text = await response.text(); - const err: any = new Error(`Cloudflare browser rendering failed: ${response.status} ${response.statusText} ${text}`.trim()); - err.status = response.status; - throw err; - } - - return { - parsed: await response.json() as any, - }; + async fetchBrowserRenderedHTML(input: { url: string }) { + if (!(this.accountId && this.apiKey)) { + throw new Error( + "CLOUD_FLARE_API_KEY must be configured as ACCOUNT_ID:API_KEY for browser rendering.", + ); } + + const response = await fetch( + `https://api.cloudflare.com/client/v4/accounts/${this.accountId}/browser-rendering/fetch`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + }, + body: JSON.stringify(input), + }, + ); + + if (!response.ok) { + const text = await response.text(); + const err: any = new Error( + `Cloudflare browser rendering failed: ${response.status} ${response.statusText} ${text}`.trim(), + ); + err.status = response.status; + throw err; + } + + return { + parsed: (await response.json()) as any, + }; + } } diff --git a/src/shared/3rd-party/serper-search.ts b/src/shared/3rd-party/serper-search.ts index 024b303..daad359 100644 --- a/src/shared/3rd-party/serper-search.ts +++ b/src/shared/3rd-party/serper-search.ts @@ -1,113 +1,125 @@ -type SearchVariant = 'search' | 'images' | 'news'; +type SearchVariant = "search" | "images" | "news"; export type SerperSearchQueryParams = { - q: string; - page?: number; - num?: number; - start?: number; - gl?: string; - hl?: string; - location?: string; - [k: string]: any; + q: string; + page?: number; + num?: number; + start?: number; + gl?: string; + hl?: string; + location?: string; + [k: string]: any; }; export type SerperWebSearchResponse = { - organic: Array>; + organic: Array>; }; export type SerperImageSearchResponse = { - images: Array>; + images: Array>; }; export type SerperNewsSearchResponse = { - news: Array>; + news: Array>; }; export const WORLD_COUNTRIES: Record = { - US: 'United States', - CN: 'China', - JP: 'Japan', - DE: 'Germany', - FR: 'France', - GB: 'United Kingdom', - IN: 'India', - CA: 'Canada', - AU: 'Australia', - SG: 'Singapore', + US: "United States", + CN: "China", + JP: "Japan", + DE: "Germany", + FR: "France", + GB: "United Kingdom", + IN: "India", + CA: "Canada", + AU: "Australia", + SG: "Singapore", }; export const WORLD_LANGUAGES = [ - { code: 'en', name: 'English' }, - { code: 'zh', name: 'Chinese' }, - { code: 'ja', name: 'Japanese' }, - { code: 'de', name: 'German' }, - { code: 'fr', name: 'French' }, - { code: 'es', name: 'Spanish' }, + { code: "en", name: "English" }, + { code: "zh", name: "Chinese" }, + { code: "ja", name: "Japanese" }, + { code: "de", name: "German" }, + { code: "fr", name: "French" }, + { code: "es", name: "Spanish" }, ]; class SerperBaseHTTP { - constructor( - protected apiKey: string, - protected provider: 'google' | 'bing', - ) { } + constructor( + protected apiKey: string, + protected provider: "google" | "bing", + ) {} - protected getBaseUrl(variant: SearchVariant) { - const host = this.provider === 'bing' ? 'https://bing.serper.dev' : 'https://google.serper.dev'; - return `${host}/${variant}`; + protected getBaseUrl(variant: SearchVariant) { + const host = + this.provider === "bing" + ? "https://bing.serper.dev" + : "https://google.serper.dev"; + return `${host}/${variant}`; + } + + protected async request( + variant: SearchVariant, + query: SerperSearchQueryParams, + ) { + if (!this.apiKey) { + throw new Error( + `SERPER_SEARCH_API_KEY is required for ${this.provider} ${variant} search.`, + ); } - protected async request(variant: SearchVariant, query: SerperSearchQueryParams) { - if (!this.apiKey) { - throw new Error(`SERPER_SEARCH_API_KEY is required for ${this.provider} ${variant} search.`); - } + const response = await fetch(this.getBaseUrl(variant), { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-API-KEY": this.apiKey, + }, + body: JSON.stringify(query), + }); - const response = await fetch(this.getBaseUrl(variant), { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'X-API-KEY': this.apiKey, - }, - body: JSON.stringify(query), - }); - - if (!response.ok) { - const text = await response.text(); - const err: any = new Error(`Serper ${this.provider} ${variant} search failed: ${response.status} ${response.statusText} ${text}`.trim()); - err.status = response.status; - throw err; - } - - return response.json() as Promise; + if (!response.ok) { + const text = await response.text(); + const err: any = new Error( + `Serper ${this.provider} ${variant} search failed: ${response.status} ${response.statusText} ${text}`.trim(), + ); + err.status = response.status; + throw err; } - async webSearch(query: SerperSearchQueryParams) { - const parsed = await this.request('search', query); + return response.json() as Promise; + } - return { parsed }; - } + async webSearch(query: SerperSearchQueryParams) { + const parsed = await this.request("search", query); - async imageSearch(query: SerperSearchQueryParams) { - const parsed = await this.request('images', query); + return { parsed }; + } - return { parsed }; - } + async imageSearch(query: SerperSearchQueryParams) { + const parsed = await this.request( + "images", + query, + ); - async newsSearch(query: SerperSearchQueryParams) { - const parsed = await this.request('news', query); + return { parsed }; + } - return { parsed }; - } + async newsSearch(query: SerperSearchQueryParams) { + const parsed = await this.request("news", query); + + return { parsed }; + } } export class SerperGoogleHTTP extends SerperBaseHTTP { - constructor(apiKey: string) { - super(apiKey, 'google'); - } + constructor(apiKey: string) { + super(apiKey, "google"); + } } export class SerperBingHTTP extends SerperBaseHTTP { - constructor(apiKey: string) { - super(apiKey, 'bing'); - } + constructor(apiKey: string) { + super(apiKey, "bing"); + } } - diff --git a/src/shared/db/api-roll.ts b/src/shared/db/api-roll.ts index 56146fe..1b0cf94 100644 --- a/src/shared/db/api-roll.ts +++ b/src/shared/db/api-roll.ts @@ -1,36 +1,35 @@ -import { Also, Prop } from 'civkit'; -import { FirestoreRecord } from '../lib/firestore'; +import { Also, Prop } from "civkit"; +import { FirestoreRecord } from "../lib/firestore"; export enum API_CALL_STATUS { - SUCCESS = 'SUCCESS', - RATE_LIMITED = 'RATE_LIMITED', - FAILED = 'FAILED', + SUCCESS = "SUCCESS", + RATE_LIMITED = "RATE_LIMITED", + FAILED = "FAILED", } @Also({ - dictOf: Object, + dictOf: Object, }) export class ApiRollRecord extends FirestoreRecord { - static override collectionName = 'api-rolls'; + static override collectionName = "api-rolls"; - override _id!: string; + override _id!: string; - @Prop() - uid?: string; + @Prop() + uid?: string; - @Prop() - ip?: string; + @Prop() + ip?: string; - @Prop({ arrayOf: String }) - tags?: string[]; + @Prop({ arrayOf: String }) + tags?: string[]; - @Prop() - status?: API_CALL_STATUS; + @Prop() + status?: API_CALL_STATUS; - @Prop() - chargeAmount?: number; + @Prop() + chargeAmount?: number; - @Prop() - createdAt!: Date; + @Prop() + createdAt!: Date; } - diff --git a/src/shared/index.ts b/src/shared/index.ts index b5d2fa0..fd29c89 100644 --- a/src/shared/index.ts +++ b/src/shared/index.ts @@ -1,2 +1 @@ -export { ServiceBadAttemptError } from '../services/errors'; - +export { ServiceBadAttemptError } from "../services/errors"; diff --git a/src/shared/lib/errors.ts b/src/shared/lib/errors.ts index 12fb2e1..282f117 100644 --- a/src/shared/lib/errors.ts +++ b/src/shared/lib/errors.ts @@ -1,8 +1,7 @@ export { - SecurityCompromiseError, - ServiceBadApproachError, - ServiceBadAttemptError, - ServiceCrashedError, - ServiceNodeResourceDrainError, -} from '../../services/errors'; - + SecurityCompromiseError, + ServiceBadApproachError, + ServiceBadAttemptError, + ServiceCrashedError, + ServiceNodeResourceDrainError, +} from "../../services/errors"; diff --git a/src/shared/lib/firestore.ts b/src/shared/lib/firestore.ts index a510cd7..ca42883 100644 --- a/src/shared/lib/firestore.ts +++ b/src/shared/lib/firestore.ts @@ -1,316 +1,376 @@ -import { AutoCastable } from 'civkit/civ-rpc'; -import { randomUUID } from 'crypto'; -import fs from 'fs'; -import fsp from 'fs/promises'; -import path from 'path'; +import { AutoCastable } from "civkit/civ-rpc"; +import { randomUUID } from "crypto"; +import fs from "fs"; +import fsp from "fs/promises"; +import path from "path"; -type WhereOp = '==' | '>=' | '<=' | '>' | '<'; -type OrderDirection = 'asc' | 'desc'; +type WhereOp = "==" | ">=" | "<=" | ">" | "<"; +type OrderDirection = "asc" | "desc"; type IncrementOp = { - __op: 'increment'; - value: number; + __op: "increment"; + value: number; }; type QueryFilter = { - field: string; - op: WhereOp; - value: any; + field: string; + op: WhereOp; + value: any; }; type QueryOrder = { - field: string; - direction: OrderDirection; + field: string; + direction: OrderDirection; }; function reviveJson(_key: string, value: any) { - if (value && typeof value === 'object' && value.__type === 'Date') { - return new Date(value.value); - } + if (value && typeof value === "object" && value.__type === "Date") { + return new Date(value.value); + } - return value; + return value; } function serializeJson(_key: string, value: any) { - if (value instanceof Date) { - return { - __type: 'Date', - value: value.toISOString(), - }; - } + if (value instanceof Date) { + return { + __type: "Date", + value: value.toISOString(), + }; + } - return value; + return value; } function getByPath(input: any, pathText: string) { - return pathText.split('.').reduce((acc, key) => acc?.[key], input); + return pathText.split(".").reduce((acc, key) => acc?.[key], input); } function setByPath(input: any, pathText: string, value: any) { - const parts = pathText.split('.'); - const last = parts.pop()!; - let cursor = input; - for (const part of parts) { - cursor[part] ??= {}; - cursor = cursor[part]; - } + const parts = pathText.split("."); + const last = parts.pop()!; + let cursor = input; + for (const part of parts) { + cursor[part] ??= {}; + cursor = cursor[part]; + } - cursor[last] = value; + cursor[last] = value; } function applyPatch(base: any, patch: any) { - const draft = structuredClone(base || {}); + const draft = structuredClone(base || {}); - for (const [key, value] of Object.entries(patch || {})) { - const currentValue = getByPath(draft, key); - if (value && typeof value === 'object' && (value as IncrementOp).__op === 'increment') { - setByPath(draft, key, (Number(currentValue) || 0) + (value as IncrementOp).value); - continue; - } - - setByPath(draft, key, value); + for (const [key, value] of Object.entries(patch || {})) { + const currentValue = getByPath(draft, key); + if ( + value && + typeof value === "object" && + (value as IncrementOp).__op === "increment" + ) { + setByPath( + draft, + key, + (Number(currentValue) || 0) + (value as IncrementOp).value, + ); + continue; } - return draft; + setByPath(draft, key, value); + } + + return draft; } class LocalStore { - rootDir = path.resolve(process.env.LOCAL_DB_ROOT || '.cache/xread/db'); + rootDir = path.resolve(process.env.LOCAL_DB_ROOT || ".cache/xread/db"); - constructor() { - fs.mkdirSync(this.rootDir, { recursive: true }); - } + constructor() { + fs.mkdirSync(this.rootDir, { recursive: true }); + } - filePath(collectionName: string) { - return path.join(this.rootDir, `${collectionName}.json`); - } + filePath(collectionName: string) { + return path.join(this.rootDir, `${collectionName}.json`); + } - async readCollection(collectionName: string) { - const filePath = this.filePath(collectionName); - try { - const content = await fsp.readFile(filePath, 'utf8'); - return JSON.parse(content, reviveJson) as Record; - } catch (err: any) { - if (err?.code === 'ENOENT') { - return {}; - } - throw err; - } + async readCollection(collectionName: string) { + const filePath = this.filePath(collectionName); + try { + const content = await fsp.readFile(filePath, "utf8"); + return JSON.parse(content, reviveJson) as Record; + } catch (err: any) { + if (err?.code === "ENOENT") { + return {}; + } + throw err; } + } - async writeCollection(collectionName: string, payload: Record) { - const filePath = this.filePath(collectionName); - await fsp.mkdir(path.dirname(filePath), { recursive: true }); - await fsp.writeFile(filePath, JSON.stringify(payload, serializeJson, 2), 'utf8'); - } + async writeCollection(collectionName: string, payload: Record) { + const filePath = this.filePath(collectionName); + await fsp.mkdir(path.dirname(filePath), { recursive: true }); + await fsp.writeFile( + filePath, + JSON.stringify(payload, serializeJson, 2), + "utf8", + ); + } } const LOCAL_STORE = new LocalStore(); -export class LocalDocRef { - constructor( - protected recordType: T, - readonly id: string, - ) { } +export class LocalDocRef< + T extends typeof FirestoreRecord = typeof FirestoreRecord, +> { + constructor( + protected recordType: T, + readonly id: string, + ) {} - async get() { - const bucket = await LOCAL_STORE.readCollection(this.recordType.collectionName); - const raw = bucket[this.id]; - if (!raw) { - return undefined; - } - - return this.recordType.from(raw); + async get() { + const bucket = await LOCAL_STORE.readCollection( + this.recordType.collectionName, + ); + const raw = bucket[this.id]; + if (!raw) { + return undefined; } - async set(payload: any, options?: { merge?: boolean }) { - const bucket = await LOCAL_STORE.readCollection(this.recordType.collectionName); - const previous = bucket[this.id]; - const merged = options?.merge ? applyPatch(previous, payload) : { ...payload }; - merged._id ??= this.id; - bucket[this.id] = merged; - await LOCAL_STORE.writeCollection(this.recordType.collectionName, bucket); + return this.recordType.from(raw); + } - return this.recordType.from(merged); - } + async set(payload: any, options?: { merge?: boolean }) { + const bucket = await LOCAL_STORE.readCollection( + this.recordType.collectionName, + ); + const previous = bucket[this.id]; + const merged = options?.merge + ? applyPatch(previous, payload) + : { ...payload }; + merged._id ??= this.id; + bucket[this.id] = merged; + await LOCAL_STORE.writeCollection(this.recordType.collectionName, bucket); - async update(payload: any) { - const bucket = await LOCAL_STORE.readCollection(this.recordType.collectionName); - const previous = bucket[this.id] || { _id: this.id }; - const merged = applyPatch(previous, payload); - merged._id ??= this.id; - bucket[this.id] = merged; - await LOCAL_STORE.writeCollection(this.recordType.collectionName, bucket); + return this.recordType.from(merged); + } - return this.recordType.from(merged); - } + async update(payload: any) { + const bucket = await LOCAL_STORE.readCollection( + this.recordType.collectionName, + ); + const previous = bucket[this.id] || { _id: this.id }; + const merged = applyPatch(previous, payload); + merged._id ??= this.id; + bucket[this.id] = merged; + await LOCAL_STORE.writeCollection(this.recordType.collectionName, bucket); + + return this.recordType.from(merged); + } } -export class LocalQuery { - constructor( - protected recordType: T, - protected filters: QueryFilter[] = [], - protected orders: QueryOrder[] = [], - protected limitCount?: number, - ) { } +export class LocalQuery< + T extends typeof FirestoreRecord = typeof FirestoreRecord, +> { + constructor( + protected recordType: T, + protected filters: QueryFilter[] = [], + protected orders: QueryOrder[] = [], + protected limitCount?: number, + ) {} - where(field: string, op: WhereOp, value: any) { - return new LocalQuery(this.recordType, [...this.filters, { field, op, value }], this.orders, this.limitCount); + where(field: string, op: WhereOp, value: any) { + return new LocalQuery( + this.recordType, + [...this.filters, { field, op, value }], + this.orders, + this.limitCount, + ); + } + + orderBy(field: string, direction: OrderDirection = "asc") { + return new LocalQuery( + this.recordType, + this.filters, + [...this.orders, { field, direction }], + this.limitCount, + ); + } + + limit(count: number) { + return new LocalQuery(this.recordType, this.filters, this.orders, count); + } + + async exec() { + const bucket = await LOCAL_STORE.readCollection( + this.recordType.collectionName, + ); + let values = Object.values(bucket); + + for (const filter of this.filters) { + values = values.filter((entry) => { + const left = getByPath(entry, filter.field); + const right = filter.value; + switch (filter.op) { + case "==": + return left === right; + case ">=": + return left >= right; + case "<=": + return left <= right; + case ">": + return left > right; + case "<": + return left < right; + default: + return false; + } + }); } - orderBy(field: string, direction: OrderDirection = 'asc') { - return new LocalQuery(this.recordType, this.filters, [...this.orders, { field, direction }], this.limitCount); - } - - limit(count: number) { - return new LocalQuery(this.recordType, this.filters, this.orders, count); - } - - async exec() { - const bucket = await LOCAL_STORE.readCollection(this.recordType.collectionName); - let values = Object.values(bucket); - - for (const filter of this.filters) { - values = values.filter((entry) => { - const left = getByPath(entry, filter.field); - const right = filter.value; - switch (filter.op) { - case '==': - return left === right; - case '>=': - return left >= right; - case '<=': - return left <= right; - case '>': - return left > right; - case '<': - return left < right; - default: - return false; - } - }); + for (const order of this.orders.slice().reverse()) { + values.sort((a, b) => { + const left = getByPath(a, order.field); + const right = getByPath(b, order.field); + const factor = order.direction === "desc" ? -1 : 1; + if (left === right) { + return 0; } - for (const order of this.orders.slice().reverse()) { - values.sort((a, b) => { - const left = getByPath(a, order.field); - const right = getByPath(b, order.field); - const factor = order.direction === 'desc' ? -1 : 1; - if (left === right) { - return 0; - } - - return left > right ? factor : -factor; - }); - } - - if (typeof this.limitCount === 'number') { - values = values.slice(0, this.limitCount); - } - - return values.map((entry) => this.recordType.from(entry)); + return left > right ? factor : -factor; + }); } + + if (typeof this.limitCount === "number") { + values = values.slice(0, this.limitCount); + } + + return values.map((entry) => this.recordType.from(entry)); + } } -export class LocalCollectionRef extends LocalQuery { - constructor(recordType: T) { - super(recordType); - } +export class LocalCollectionRef< + T extends typeof FirestoreRecord = typeof FirestoreRecord, +> extends LocalQuery { + constructor(recordType: T) { + super(recordType); + } - doc(id?: string) { - return new LocalDocRef(this.recordType, id || randomUUID()); - } + doc(id?: string) { + return new LocalDocRef(this.recordType, id || randomUUID()); + } } class LocalBatch { - protected tasks: Array<() => Promise> = []; + protected tasks: Array<() => Promise> = []; - set(ref: LocalDocRef, payload: any, options?: { merge?: boolean }) { - this.tasks.push(() => ref.set(payload, options)); - } + set(ref: LocalDocRef, payload: any, options?: { merge?: boolean }) { + this.tasks.push(() => ref.set(payload, options)); + } - update(ref: LocalDocRef, payload: any) { - this.tasks.push(() => ref.update(payload)); - } + update(ref: LocalDocRef, payload: any) { + this.tasks.push(() => ref.update(payload)); + } - async commit() { - await Promise.all(this.tasks.map((task) => task())); - } + async commit() { + await Promise.all(this.tasks.map((task) => task())); + } } class LocalTransaction { - async get(ref: LocalDocRef) { - return ref.get(); - } + async get(ref: LocalDocRef) { + return ref.get(); + } - set(ref: LocalDocRef, payload: any, options?: { merge?: boolean }) { - return ref.set(payload, options); - } + set(ref: LocalDocRef, payload: any, options?: { merge?: boolean }) { + return ref.set(payload, options); + } - update(ref: LocalDocRef, payload: any) { - return ref.update(payload); - } + update(ref: LocalDocRef, payload: any) { + return ref.update(payload); + } } class LocalDB { - batch() { - return new LocalBatch(); - } + batch() { + return new LocalBatch(); + } - async runTransaction(handler: (transaction: LocalTransaction) => Promise) { - return handler(new LocalTransaction()); - } + async runTransaction( + handler: (transaction: LocalTransaction) => Promise, + ) { + return handler(new LocalTransaction()); + } } const LOCAL_DB = new LocalDB(); export class FirestoreRecord extends AutoCastable { - static collectionName = 'records'; + static collectionName = "records"; - _id!: string; + _id!: string; - static get COLLECTION() { - return new LocalCollectionRef(this); + static get COLLECTION() { + return new LocalCollectionRef(this); + } + + static get DB() { + return LOCAL_DB; + } + + static OPS = { + increment(value: number): IncrementOp { + return { + __op: "increment", + value, + }; + }, + }; + + static async fromFirestore( + this: T, + id: string, + ) { + if (!id) { + return undefined; } - static get DB() { - return LOCAL_DB; - } + return this.COLLECTION.doc(id).get() as Promise< + InstanceType | undefined + >; + } - static OPS = { - increment(value: number): IncrementOp { - return { - __op: 'increment', - value, - }; - }, - }; + static async fromFirestoreQuery( + this: T, + query: LocalQuery, + ) { + return query.exec() as Promise>>; + } - static async fromFirestore(this: T, id: string) { - if (!id) { - return undefined; - } + static async save( + this: T, + input: any, + id?: string, + options?: { merge?: boolean }, + ) { + const payload = + input instanceof this ? input.degradeForFireStore() : { ...input }; + const resolvedId = id || payload._id || randomUUID(); + payload._id = resolvedId; - return this.COLLECTION.doc(id).get() as Promise | undefined>; - } + return this.COLLECTION.doc(resolvedId).set(payload, options) as Promise< + InstanceType + >; + } - static async fromFirestoreQuery(this: T, query: LocalQuery) { - return query.exec() as Promise>>; - } + async save(options?: { merge?: boolean }) { + return (this.constructor as typeof FirestoreRecord).save( + this, + this._id, + options, + ); + } - static async save(this: T, input: any, id?: string, options?: { merge?: boolean }) { - const payload = input instanceof this ? input.degradeForFireStore() : { ...input }; - const resolvedId = id || payload._id || randomUUID(); - payload._id = resolvedId; - - return this.COLLECTION.doc(resolvedId).set(payload, options) as Promise>; - } - - async save(options?: { merge?: boolean }) { - return (this.constructor as typeof FirestoreRecord).save(this, this._id, options); - } - - degradeForFireStore() { - return { ...this }; - } + degradeForFireStore() { + return { ...this }; + } } diff --git a/src/shared/services/async-context.ts b/src/shared/services/async-context.ts index bb45de6..52589ec 100644 --- a/src/shared/services/async-context.ts +++ b/src/shared/services/async-context.ts @@ -1,2 +1 @@ -export { AsyncLocalContext as AsyncContext } from '../../services/async-context'; - +export { AsyncLocalContext as AsyncContext } from "../../services/async-context"; diff --git a/src/shared/services/common-iminterrogate.ts b/src/shared/services/common-iminterrogate.ts index 501fd4e..bb6d359 100644 --- a/src/shared/services/common-iminterrogate.ts +++ b/src/shared/services/common-iminterrogate.ts @@ -1,15 +1,17 @@ -import { AsyncService } from 'civkit/async-service'; -import { singleton } from 'tsyringe'; -import { ServiceDisabledError } from '../../services/errors'; +import { AsyncService } from "civkit/async-service"; +import { singleton } from "tsyringe"; +import { ServiceDisabledError } from "../../services/errors"; @singleton() export class ImageInterrogationManager extends AsyncService { - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } - async interrogate(model: string, _input?: unknown): Promise { - throw new ServiceDisabledError(`Image interrogation feature '${model}' is not enabled in this standalone build.`); - } + async interrogate(model: string, _input?: unknown): Promise { + throw new ServiceDisabledError( + `Image interrogation feature '${model}' is not enabled in this standalone build.`, + ); + } } diff --git a/src/shared/services/common-llm.ts b/src/shared/services/common-llm.ts index dc86905..78c5bd7 100644 --- a/src/shared/services/common-llm.ts +++ b/src/shared/services/common-llm.ts @@ -1,15 +1,17 @@ -import { AsyncService } from 'civkit/async-service'; -import { singleton } from 'tsyringe'; -import { ServiceDisabledError } from '../../services/errors'; +import { AsyncService } from "civkit/async-service"; +import { singleton } from "tsyringe"; +import { ServiceDisabledError } from "../../services/errors"; @singleton() export class LLMManager extends AsyncService { - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } - async *iterRun(model: string, _input?: unknown): AsyncGenerator { - throw new ServiceDisabledError(`LLM feature '${model}' is not enabled in this standalone build.`); - } + async *iterRun(model: string, _input?: unknown): AsyncGenerator { + throw new ServiceDisabledError( + `LLM feature '${model}' is not enabled in this standalone build.`, + ); + } } diff --git a/src/shared/services/firebase-storage-bucket.ts b/src/shared/services/firebase-storage-bucket.ts index 42366d6..a2eb2ea 100644 --- a/src/shared/services/firebase-storage-bucket.ts +++ b/src/shared/services/firebase-storage-bucket.ts @@ -1,53 +1,56 @@ -import { AsyncService } from 'civkit/async-service'; -import { singleton } from 'tsyringe'; -import path from 'path'; -import fs from 'fs/promises'; -import { pathToFileURL } from 'url'; -import { SecretExposer } from './secrets'; +import { AsyncService } from "civkit/async-service"; +import { singleton } from "tsyringe"; +import path from "path"; +import fs from "fs/promises"; +import { pathToFileURL } from "url"; +import { SecretExposer } from "./secrets"; type SaveOptions = { + contentType?: string; + metadata?: { contentType?: string; - metadata?: { - contentType?: string; - }; + }; }; @singleton() export class FirebaseStorageBucketControl extends AsyncService { - rootDir!: string; + rootDir!: string; - constructor( - protected secretExposer: SecretExposer, - ) { - super(...arguments); - } + constructor(protected secretExposer: SecretExposer) { + super(...arguments); + } - override async init() { - await this.dependencyReady(); - this.rootDir = path.resolve(this.secretExposer.STORAGE_ROOT || '.cache/xread/storage'); - await fs.mkdir(this.rootDir, { recursive: true }); - this.emit('ready'); - } + override async init() { + await this.dependencyReady(); + this.rootDir = path.resolve( + this.secretExposer.STORAGE_ROOT || ".cache/xread/storage", + ); + await fs.mkdir(this.rootDir, { recursive: true }); + this.emit("ready"); + } - protected resolvePath(key: string) { - const safeKey = key.replace(/\\/g, '/').replace(/^\/+/, ''); - return path.resolve(this.rootDir, safeKey); - } + protected resolvePath(key: string) { + const safeKey = key.replace(/\\/g, "/").replace(/^\/+/, ""); + return path.resolve(this.rootDir, safeKey); + } - async saveFile(key: string, content: Buffer | Uint8Array, _options?: SaveOptions) { - const fullPath = this.resolvePath(key); - await fs.mkdir(path.dirname(fullPath), { recursive: true }); - await fs.writeFile(fullPath, content); + async saveFile( + key: string, + content: Buffer | Uint8Array, + _options?: SaveOptions, + ) { + const fullPath = this.resolvePath(key); + await fs.mkdir(path.dirname(fullPath), { recursive: true }); + await fs.writeFile(fullPath, content); - return { fullPath }; - } + return { fullPath }; + } - async downloadFile(key: string) { - return fs.readFile(this.resolvePath(key)); - } + async downloadFile(key: string) { + return fs.readFile(this.resolvePath(key)); + } - async signDownloadUrl(key: string, _expiresAt?: number) { - return pathToFileURL(this.resolvePath(key)).href; - } + async signDownloadUrl(key: string, _expiresAt?: number) { + return pathToFileURL(this.resolvePath(key)).href; + } } - diff --git a/src/shared/services/proxy-provider.ts b/src/shared/services/proxy-provider.ts index 3871876..16f4227 100644 --- a/src/shared/services/proxy-provider.ts +++ b/src/shared/services/proxy-provider.ts @@ -1,54 +1,55 @@ -import { AsyncService } from 'civkit/async-service'; -import { singleton } from 'tsyringe'; -import { SecretExposer } from './secrets'; -import { ServiceDisabledError } from '../../services/errors'; +import { AsyncService } from "civkit/async-service"; +import { singleton } from "tsyringe"; +import { SecretExposer } from "./secrets"; +import { ServiceDisabledError } from "../../services/errors"; @singleton() export class ProxyProviderService extends AsyncService { - protected proxies: URL[] = []; + protected proxies: URL[] = []; - constructor( - protected secretExposer: SecretExposer, - ) { - super(...arguments); + constructor(protected secretExposer: SecretExposer) { + super(...arguments); + } + + override async init() { + await this.dependencyReady(); + this.proxies = this.secretExposer.LOCAL_PROXY_URLS.split(",") + .map((x) => x.trim()) + .filter(Boolean) + .map((x) => new URL(x)); + this.emit("ready"); + } + + supports(input?: string) { + if (!input || input === "auto" || input === "any" || input === "none") { + return true; } - override async init() { - await this.dependencyReady(); - this.proxies = this.secretExposer.LOCAL_PROXY_URLS - .split(',') - .map((x) => x.trim()) - .filter(Boolean) - .map((x) => new URL(x)); - this.emit('ready'); + return /^[a-z]{2}$/i.test(input); + } + + isConfigured() { + return this.proxies.length > 0; + } + + async alloc(input?: string) { + if (input === "none") { + throw new ServiceDisabledError( + `Proxy allocation is disabled for '${input}'.`, + ); } - supports(input?: string) { - if (!input || input === 'auto' || input === 'any' || input === 'none') { - return true; - } - - return /^[a-z]{2}$/i.test(input); + const picked = this.proxies[0]; + if (!picked) { + throw new ServiceDisabledError( + "Proxy allocation is not configured in this standalone build.", + ); } - isConfigured() { - return this.proxies.length > 0; - } + return picked; + } - async alloc(input?: string) { - if (input === 'none') { - throw new ServiceDisabledError(`Proxy allocation is disabled for '${input}'.`); - } - - const picked = this.proxies[0]; - if (!picked) { - throw new ServiceDisabledError('Proxy allocation is not configured in this standalone build.'); - } - - return picked; - } - - async *iterAlloc(input?: string) { - yield await this.alloc(input); - } + async *iterAlloc(input?: string) { + yield await this.alloc(input); + } } diff --git a/src/shared/services/rate-limit.ts b/src/shared/services/rate-limit.ts index 482815f..ae160e6 100644 --- a/src/shared/services/rate-limit.ts +++ b/src/shared/services/rate-limit.ts @@ -1,109 +1,147 @@ -import { AsyncService } from 'civkit/async-service'; -import { ApplicationError, AutoCastable } from 'civkit/civ-rpc'; -import { singleton } from 'tsyringe'; -import { ApiRollRecord, API_CALL_STATUS } from '../db/api-roll'; +import { AsyncService } from "civkit/async-service"; +import { ApplicationError, AutoCastable } from "civkit/civ-rpc"; +import { singleton } from "tsyringe"; +import { ApiRollRecord, API_CALL_STATUS } from "../db/api-roll"; type SubjectLimit = { - at: number; + at: number; }; @singleton() export class RateLimitControl extends AsyncService { - protected hits = new Map(); + protected hits = new Map(); - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } - protected prune(key: string, windowMs: number) { - const now = Date.now(); - const entries = (this.hits.get(key) || []).filter((entry) => (now - entry.at) < windowMs); - this.hits.set(key, entries); + protected prune(key: string, windowMs: number) { + const now = Date.now(); + const entries = (this.hits.get(key) || []).filter( + (entry) => now - entry.at < windowMs, + ); + this.hits.set(key, entries); - return entries; - } + return entries; + } - protected async checkAndRecord(subject: string, tags: string[], descs: RateLimitDesc[]) { - const now = Date.now(); - for (const desc of descs) { - const key = `${subject}:${tags.join(',')}:${desc.periodSeconds}:${desc.occurrence}`; - const entries = this.prune(key, desc.periodSeconds * 1000); - if (entries.length >= desc.occurrence) { - const oldestRelevant = entries[0]; - const retryAfterMs = Math.max(1000, desc.periodSeconds * 1000 - (now - oldestRelevant.at)); - throw RateLimitTriggeredError.from({ - message: `Rate limit exceeded for ${subject}`, - retryAfter: Math.ceil(retryAfterMs / 1000), - retryAfterDate: new Date(now + retryAfterMs), - }); - } - - entries.push({ at: now }); - this.hits.set(key, entries); - } - } - - async simpleRPCUidBasedLimit(_rpcReflect: any, uid: string, tags: string[], ...descs: RateLimitDesc[]) { - const effectiveDescs = descs.length ? descs : [RateLimitDesc.from({ occurrence: 60, periodSeconds: 60 })]; - await this.checkAndRecord(`uid:${uid}`, tags, effectiveDescs); - - return ApiRollRecord.from({ - _id: `${Date.now()}-${Math.random()}`, - uid, - tags, - status: API_CALL_STATUS.SUCCESS, - createdAt: new Date(), + protected async checkAndRecord( + subject: string, + tags: string[], + descs: RateLimitDesc[], + ) { + const now = Date.now(); + for (const desc of descs) { + const key = `${subject}:${tags.join(",")}:${desc.periodSeconds}:${desc.occurrence}`; + const entries = this.prune(key, desc.periodSeconds * 1000); + if (entries.length >= desc.occurrence) { + const oldestRelevant = entries[0]; + const retryAfterMs = Math.max( + 1000, + desc.periodSeconds * 1000 - (now - oldestRelevant.at), + ); + throw RateLimitTriggeredError.from({ + message: `Rate limit exceeded for ${subject}`, + retryAfter: Math.ceil(retryAfterMs / 1000), + retryAfterDate: new Date(now + retryAfterMs), }); + } + + entries.push({ at: now }); + this.hits.set(key, entries); } + } - async simpleRpcIPBasedLimit(_rpcReflect: any, ip: string, tags: string[], limits?: [Date, number] | Array<[Date, number]>) { - const normalized = Array.isArray(limits?.[0]) ? limits as Array<[Date, number]> : (limits ? [limits as [Date, number]] : []); - const descs = normalized.map(([date, occurrence]) => { - const seconds = Math.max(1, Math.ceil((Date.now() - date.valueOf()) / 1000)); - return RateLimitDesc.from({ - occurrence, - periodSeconds: seconds, - }); - }); + async simpleRPCUidBasedLimit( + _rpcReflect: any, + uid: string, + tags: string[], + ...descs: RateLimitDesc[] + ) { + const effectiveDescs = descs.length + ? descs + : [RateLimitDesc.from({ occurrence: 60, periodSeconds: 60 })]; + await this.checkAndRecord(`uid:${uid}`, tags, effectiveDescs); - await this.checkAndRecord(`ip:${ip}`, tags, descs.length ? descs : [RateLimitDesc.from({ occurrence: 20, periodSeconds: 60 })]); + return ApiRollRecord.from({ + _id: `${Date.now()}-${Math.random()}`, + uid, + tags, + status: API_CALL_STATUS.SUCCESS, + createdAt: new Date(), + }); + } - return ApiRollRecord.from({ - _id: `${Date.now()}-${Math.random()}`, - ip, - tags, - status: API_CALL_STATUS.SUCCESS, - createdAt: new Date(), - }); - } + async simpleRpcIPBasedLimit( + _rpcReflect: any, + ip: string, + tags: string[], + limits?: [Date, number] | Array<[Date, number]>, + ) { + const normalized = Array.isArray(limits?.[0]) + ? (limits as Array<[Date, number]>) + : limits + ? [limits as [Date, number]] + : []; + const descs = normalized.map(([date, occurrence]) => { + const seconds = Math.max( + 1, + Math.ceil((Date.now() - date.valueOf()) / 1000), + ); + return RateLimitDesc.from({ + occurrence, + periodSeconds: seconds, + }); + }); - record(input: Partial) { - return ApiRollRecord.from({ - createdAt: new Date(), - status: API_CALL_STATUS.SUCCESS, - ...input, - }); - } + await this.checkAndRecord( + `ip:${ip}`, + tags, + descs.length + ? descs + : [RateLimitDesc.from({ occurrence: 20, periodSeconds: 60 })], + ); + + return ApiRollRecord.from({ + _id: `${Date.now()}-${Math.random()}`, + ip, + tags, + status: API_CALL_STATUS.SUCCESS, + createdAt: new Date(), + }); + } + + record(input: Partial) { + return ApiRollRecord.from({ + createdAt: new Date(), + status: API_CALL_STATUS.SUCCESS, + ...input, + }); + } } export class RateLimitDesc extends AutoCastable { - occurrence!: number; - periodSeconds!: number; + occurrence!: number; + periodSeconds!: number; - isEffective() { - return Boolean(this.occurrence && this.periodSeconds); - } + isEffective() { + return Boolean(this.occurrence && this.periodSeconds); + } } export class RateLimitTriggeredError extends ApplicationError { - retryAfter?: number; - retryAfterDate?: Date; + retryAfter?: number; + retryAfterDate?: Date; - static override from(input: Partial & { message?: string }) { - const err = new RateLimitTriggeredError(input.message || 'Rate limit exceeded'); - Object.assign(err, input); + static override from( + input: Partial & { message?: string }, + ) { + const err = new RateLimitTriggeredError( + input.message || "Rate limit exceeded", + ); + Object.assign(err, input); - return err; - } + return err; + } } diff --git a/src/shared/services/secrets.ts b/src/shared/services/secrets.ts index 54cdd42..a67c45d 100644 --- a/src/shared/services/secrets.ts +++ b/src/shared/services/secrets.ts @@ -1,53 +1,56 @@ -import { AsyncService } from 'civkit/async-service'; -import { singleton } from 'tsyringe'; +import { AsyncService } from "civkit/async-service"; +import { singleton } from "tsyringe"; type EnvConfig = Record & { - readonly SERPER_SEARCH_API_KEY: string; - readonly BRAVE_SEARCH_API_KEY: string; - readonly CLOUD_FLARE_API_KEY: string; - readonly LOCAL_PROXY_URLS: string; - readonly STORAGE_ROOT: string; - readonly LOCAL_DB_ROOT: string; + readonly SERPER_SEARCH_API_KEY: string; + readonly BRAVE_SEARCH_API_KEY: string; + readonly CLOUD_FLARE_API_KEY: string; + readonly LOCAL_PROXY_URLS: string; + readonly STORAGE_ROOT: string; + readonly LOCAL_DB_ROOT: string; }; -export const readEnv = (key: string) => process.env[key] || ''; +export const readEnv = (key: string) => process.env[key] || ""; -const envConfig = new Proxy({}, { +const envConfig = new Proxy( + {}, + { get(_target, prop) { - return readEnv(String(prop)); + return readEnv(String(prop)); }, -}) as EnvConfig; + }, +) as EnvConfig; @singleton() export class SecretExposer extends AsyncService { - override async init() { - await this.dependencyReady(); - this.emit('ready'); - } + override async init() { + await this.dependencyReady(); + this.emit("ready"); + } - get SERPER_SEARCH_API_KEY() { - return readEnv('SERPER_SEARCH_API_KEY'); - } + get SERPER_SEARCH_API_KEY() { + return readEnv("SERPER_SEARCH_API_KEY"); + } - get BRAVE_SEARCH_API_KEY() { - return readEnv('BRAVE_SEARCH_API_KEY'); - } + get BRAVE_SEARCH_API_KEY() { + return readEnv("BRAVE_SEARCH_API_KEY"); + } - get CLOUD_FLARE_API_KEY() { - return readEnv('CLOUD_FLARE_API_KEY'); - } + get CLOUD_FLARE_API_KEY() { + return readEnv("CLOUD_FLARE_API_KEY"); + } - get LOCAL_PROXY_URLS() { - return readEnv('LOCAL_PROXY_URLS'); - } + get LOCAL_PROXY_URLS() { + return readEnv("LOCAL_PROXY_URLS"); + } - get STORAGE_ROOT() { - return readEnv('STORAGE_ROOT'); - } + get STORAGE_ROOT() { + return readEnv("STORAGE_ROOT"); + } - get LOCAL_DB_ROOT() { - return readEnv('LOCAL_DB_ROOT'); - } + get LOCAL_DB_ROOT() { + return readEnv("LOCAL_DB_ROOT"); + } } export default envConfig; diff --git a/src/shared/utils/audition.ts b/src/shared/utils/audition.ts index 11d7355..ca50512 100644 --- a/src/shared/utils/audition.ts +++ b/src/shared/utils/audition.ts @@ -1,6 +1,5 @@ -import type { Context, Next } from 'koa'; +import type { Context, Next } from "koa"; export function getAuditionMiddleware() { - return async (_ctx: Context, next: Next) => next(); + return async (_ctx: Context, next: Next) => next(); } - diff --git a/src/shared/utils/openai.ts b/src/shared/utils/openai.ts index c40b30c..d8d51ab 100644 --- a/src/shared/utils/openai.ts +++ b/src/shared/utils/openai.ts @@ -1,8 +1,7 @@ export function countGPTToken(input?: string | null) { - if (!input) { - return 0; - } + if (!input) { + return 0; + } - return Math.max(1, Math.ceil(Buffer.byteLength(input, 'utf8') / 4)); + return Math.max(1, Math.ceil(Buffer.byteLength(input, "utf8") / 4)); } - diff --git a/src/stand-alone/crawl.ts b/src/stand-alone/crawl.ts index 852f2da..ad27e90 100644 --- a/src/stand-alone/crawl.ts +++ b/src/stand-alone/crawl.ts @@ -1,175 +1,193 @@ -import 'reflect-metadata'; -import { container, singleton } from 'tsyringe'; +import "reflect-metadata"; +import { container, singleton } from "tsyringe"; -import { KoaServer } from 'civkit/civ-rpc/koa'; -import http2 from 'http2'; -import http from 'http'; -import { CrawlerHost } from '../api/crawler'; -import { FsWalk, WalkOutEntity } from 'civkit/fswalk'; -import path from 'path'; -import fs from 'fs'; -import { mimeOfExt } from 'civkit/mime'; -import { Context, Next } from 'koa'; -import { RPCRegistry } from '../services/registry'; -import { AsyncResource } from 'async_hooks'; -import { runOnce } from 'civkit/decorators'; -import { randomUUID } from 'crypto'; -import { ThreadedServiceRegistry } from '../services/threaded'; -import { GlobalLogger } from '../services/logger'; -import { AsyncLocalContext } from '../services/async-context'; -import finalizer, { Finalizer } from '../services/finalizer'; -import koaCompress from 'koa-compress'; -import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; +import { KoaServer } from "civkit/civ-rpc/koa"; +import http2 from "http2"; +import http from "http"; +import { CrawlerHost } from "../api/crawler"; +import { FsWalk, WalkOutEntity } from "civkit/fswalk"; +import path from "path"; +import fs from "fs"; +import { mimeOfExt } from "civkit/mime"; +import { Context, Next } from "koa"; +import { RPCRegistry } from "../services/registry"; +import { AsyncResource } from "async_hooks"; +import { runOnce } from "civkit/decorators"; +import { randomUUID } from "crypto"; +import { ThreadedServiceRegistry } from "../services/threaded"; +import { GlobalLogger } from "../services/logger"; +import { AsyncLocalContext } from "../services/async-context"; +import finalizer, { Finalizer } from "../services/finalizer"; +import koaCompress from "koa-compress"; +import { STANDALONE_BOOT_TIMEOUT_MS } from "../services/boot-timeouts"; @singleton() export class CrawlStandAloneServer extends KoaServer { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - httpAlternativeServer?: typeof this['httpServer']; - assets = new Map(); + httpAlternativeServer?: (typeof this)["httpServer"]; + assets = new Map(); - constructor( - protected globalLogger: GlobalLogger, - protected registry: RPCRegistry, - protected crawlerHost: CrawlerHost, - protected threadLocal: AsyncLocalContext, - protected threads: ThreadedServiceRegistry, - ) { - super(...arguments); + constructor( + protected globalLogger: GlobalLogger, + protected registry: RPCRegistry, + protected crawlerHost: CrawlerHost, + protected threadLocal: AsyncLocalContext, + protected threads: ThreadedServiceRegistry, + ) { + super(...arguments); + } + + h2c() { + this.httpAlternativeServer = this.httpServer; + const fn = this.koaApp.callback(); + this.httpServer = http2.createServer((req, res) => { + const ar = new AsyncResource("HTTP2ServerRequest"); + ar.runInAsyncScope(fn, this.koaApp, req, res); + }); + // useResourceBasedDefaultTracker(); + + return this; + } + + override async init() { + await this.walkForAssets(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + await super.init(); + } + + override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { + return super.dependencyReady(timeoutMilliseconds); + } + + async walkForAssets() { + const files = await FsWalk.walkOut( + path.resolve(__dirname, "..", "..", "public"), + ); + + for (const file of files) { + if (file.type !== "file") { + continue; + } + this.assets.set(file.relativePath.toString(), file); + } + } + + override listen(port: number) { + const r = super.listen(port); + if (this.httpAlternativeServer) { + const altPort = port + 1; + this.httpAlternativeServer.listen(altPort, () => { + this.logger.info( + `Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`, + ); + }); } - h2c() { - this.httpAlternativeServer = this.httpServer; - const fn = this.koaApp.callback(); - this.httpServer = http2.createServer((req, res) => { - const ar = new AsyncResource('HTTP2ServerRequest'); - ar.runInAsyncScope(fn, this.koaApp, req, res); - }); - // useResourceBasedDefaultTracker(); + return r; + } - return this; - } + makeAssetsServingController() { + return (ctx: Context, next: Next) => { + const requestPath = ctx.path; + const file = requestPath.slice(1); + if (!file) { + return next(); + } - override async init() { - await this.walkForAssets(); - await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); - await super.init(); - } + const asset = this.assets.get(file); + if (asset?.type !== "file") { + return next(); + } - override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { - return super.dependencyReady(timeoutMilliseconds); - } + ctx.body = fs.createReadStream(asset.path); + ctx.type = + mimeOfExt(path.extname(asset.path.toString())) || + "application/octet-stream"; + ctx.set("Content-Length", asset.stats.size.toString()); - async walkForAssets() { - const files = await FsWalk.walkOut(path.resolve(__dirname, '..', '..', 'public')); + return; + }; + } - for (const file of files) { - if (file.type !== 'file') { - continue; + registerRoutes(): void { + this.koaApp.use( + koaCompress({ + filter(type) { + if (type.startsWith("text/")) { + return true; + } + + if ( + type.includes("application/json") || + type.includes("+json") || + type.includes("+xml") + ) { + return true; + } + + if (type.includes("application/x-ndjson")) { + return true; + } + + return false; + }, + }), + ); + this.koaApp.use(this.makeAssetsServingController()); + this.koaApp.use(this.registry.makeShimController()); + } + + // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context + // TraceId is expected to be request-bound and unique. So these two has to be distinguished. + @runOnce() + override insertAsyncHookMiddleware() { + const asyncHookMiddleware = async ( + ctx: Context, + next: () => Promise, + ) => { + const googleTraceId = ctx.get("x-cloud-trace-context").split("/")?.[0]; + this.threadLocal.setup({ + traceId: randomUUID(), + traceT0: new Date(), + googleTraceId, + }); + + return next(); + }; + + this.koaApp.use(asyncHookMiddleware); + } + + @Finalizer() + override async standDown() { + const tasks: Promise[] = []; + if (this.httpAlternativeServer?.listening) { + (this.httpAlternativeServer as http.Server).closeIdleConnections?.(); + this.httpAlternativeServer.close(); + tasks.push( + new Promise((resolve, reject) => { + this.httpAlternativeServer!.close((err) => { + if (err) { + return reject(err); } - this.assets.set(file.relativePath.toString(), file); - } + resolve(); + }); + }), + ); } - - override listen(port: number) { - const r = super.listen(port); - if (this.httpAlternativeServer) { - const altPort = port + 1; - this.httpAlternativeServer.listen(altPort, () => { - this.logger.info(`Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`); - }); - } - - return r; - } - - makeAssetsServingController() { - return (ctx: Context, next: Next) => { - const requestPath = ctx.path; - const file = requestPath.slice(1); - if (!file) { - return next(); - } - - const asset = this.assets.get(file); - if (asset?.type !== 'file') { - return next(); - } - - ctx.body = fs.createReadStream(asset.path); - ctx.type = mimeOfExt(path.extname(asset.path.toString())) || 'application/octet-stream'; - ctx.set('Content-Length', asset.stats.size.toString()); - - return; - }; - } - - registerRoutes(): void { - this.koaApp.use(koaCompress({ - filter(type) { - if (type.startsWith('text/')) { - return true; - } - - if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) { - return true; - } - - if (type.includes('application/x-ndjson')) { - return true; - } - - return false; - } - })); - this.koaApp.use(this.makeAssetsServingController()); - this.koaApp.use(this.registry.makeShimController()); - } - - // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context - // TraceId is expected to be request-bound and unique. So these two has to be distinguished. - @runOnce() - override insertAsyncHookMiddleware() { - const asyncHookMiddleware = async (ctx: Context, next: () => Promise) => { - const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0]; - this.threadLocal.setup({ - traceId: randomUUID(), - traceT0: new Date(), - googleTraceId, - }); - - return next(); - }; - - this.koaApp.use(asyncHookMiddleware); - } - - @Finalizer() - override async standDown() { - const tasks: Promise[] = []; - if (this.httpAlternativeServer?.listening) { - (this.httpAlternativeServer as http.Server).closeIdleConnections?.(); - this.httpAlternativeServer.close(); - tasks.push(new Promise((resolve, reject) => { - this.httpAlternativeServer!.close((err) => { - if (err) { - return reject(err); - } - resolve(); - }); - })); - } - tasks.push(super.standDown()); - await Promise.all(tasks); - } - + tasks.push(super.standDown()); + await Promise.all(tasks); + } } const instance = container.resolve(CrawlStandAloneServer); export default instance; -if (process.env.NODE_ENV?.includes('dry-run')) { - instance.serviceReady().then(() => finalizer.terminate()); +if (process.env.NODE_ENV?.includes("dry-run")) { + instance.serviceReady().then(() => finalizer.terminate()); } else { - instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000)); + instance + .serviceReady() + .then((s) => s.h2c().listen(parseInt(process.env.PORT || "") || 3000)); } diff --git a/src/stand-alone/search.ts b/src/stand-alone/search.ts index 88d5964..fe693d9 100644 --- a/src/stand-alone/search.ts +++ b/src/stand-alone/search.ts @@ -1,183 +1,200 @@ -import 'reflect-metadata'; -import { container, singleton } from 'tsyringe'; +import "reflect-metadata"; +import { container, singleton } from "tsyringe"; -import { KoaServer } from 'civkit/civ-rpc/koa'; -import http2 from 'http2'; -import http from 'http'; -import { SearcherHost } from '../api/searcher'; -import { FsWalk, WalkOutEntity } from 'civkit/fswalk'; -import path from 'path'; -import fs from 'fs'; -import { mimeOfExt } from 'civkit/mime'; -import { Context, Next } from 'koa'; -import { RPCRegistry } from '../services/registry'; -import { AsyncResource } from 'async_hooks'; -import { runOnce } from 'civkit/decorators'; -import { randomUUID } from 'crypto'; -import { ThreadedServiceRegistry } from '../services/threaded'; -import { GlobalLogger } from '../services/logger'; -import { AsyncLocalContext } from '../services/async-context'; -import finalizer, { Finalizer } from '../services/finalizer'; -import koaCompress from 'koa-compress'; -import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; +import { KoaServer } from "civkit/civ-rpc/koa"; +import http2 from "http2"; +import http from "http"; +import { SearcherHost } from "../api/searcher"; +import { FsWalk, WalkOutEntity } from "civkit/fswalk"; +import path from "path"; +import fs from "fs"; +import { mimeOfExt } from "civkit/mime"; +import { Context, Next } from "koa"; +import { RPCRegistry } from "../services/registry"; +import { AsyncResource } from "async_hooks"; +import { runOnce } from "civkit/decorators"; +import { randomUUID } from "crypto"; +import { ThreadedServiceRegistry } from "../services/threaded"; +import { GlobalLogger } from "../services/logger"; +import { AsyncLocalContext } from "../services/async-context"; +import finalizer, { Finalizer } from "../services/finalizer"; +import koaCompress from "koa-compress"; +import { STANDALONE_BOOT_TIMEOUT_MS } from "../services/boot-timeouts"; @singleton() export class SearchStandAloneServer extends KoaServer { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - httpAlternativeServer?: typeof this['httpServer']; - assets = new Map(); + httpAlternativeServer?: (typeof this)["httpServer"]; + assets = new Map(); - constructor( - protected globalLogger: GlobalLogger, - protected registry: RPCRegistry, - protected searcherHost: SearcherHost, - protected threadLocal: AsyncLocalContext, - protected threads: ThreadedServiceRegistry, - ) { - super(...arguments); + constructor( + protected globalLogger: GlobalLogger, + protected registry: RPCRegistry, + protected searcherHost: SearcherHost, + protected threadLocal: AsyncLocalContext, + protected threads: ThreadedServiceRegistry, + ) { + super(...arguments); + } + + h2c() { + this.httpAlternativeServer = this.httpServer; + const fn = this.koaApp.callback(); + this.httpServer = http2.createServer((req, res) => { + const ar = new AsyncResource("HTTP2ServerRequest"); + ar.runInAsyncScope(fn, this.koaApp, req, res); + }); + // useResourceBasedDefaultTracker(); + + return this; + } + + override async init() { + await this.walkForAssets(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + + for (const [k, v] of this.registry.conf.entries()) { + if (v.tags?.includes("crawl")) { + this.registry.conf.delete(k); + } } - h2c() { - this.httpAlternativeServer = this.httpServer; - const fn = this.koaApp.callback(); - this.httpServer = http2.createServer((req, res) => { - const ar = new AsyncResource('HTTP2ServerRequest'); - ar.runInAsyncScope(fn, this.koaApp, req, res); - }); - // useResourceBasedDefaultTracker(); + await super.init(); + } - return this; + override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { + return super.dependencyReady(timeoutMilliseconds); + } + + async walkForAssets() { + const files = await FsWalk.walkOut( + path.resolve(__dirname, "..", "..", "public"), + ); + + for (const file of files) { + if (file.type !== "file") { + continue; + } + this.assets.set(file.relativePath.toString(), file); + } + } + + override listen(port: number) { + const r = super.listen(port); + if (this.httpAlternativeServer) { + const altPort = port + 1; + this.httpAlternativeServer.listen(altPort, () => { + this.logger.info( + `Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`, + ); + }); } - override async init() { - await this.walkForAssets(); - await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + return r; + } - for (const [k, v] of this.registry.conf.entries()) { - if (v.tags?.includes('crawl')) { - this.registry.conf.delete(k); + makeAssetsServingController() { + return (ctx: Context, next: Next) => { + const requestPath = ctx.path; + const file = requestPath.slice(1); + if (!file) { + return next(); + } + + const asset = this.assets.get(file); + if (asset?.type !== "file") { + return next(); + } + + ctx.body = fs.createReadStream(asset.path); + ctx.type = + mimeOfExt(path.extname(asset.path.toString())) || + "application/octet-stream"; + ctx.set("Content-Length", asset.stats.size.toString()); + + return; + }; + } + + registerRoutes(): void { + this.koaApp.use( + koaCompress({ + filter(type) { + if (type.startsWith("text/")) { + return true; + } + + if ( + type.includes("application/json") || + type.includes("+json") || + type.includes("+xml") + ) { + return true; + } + + if (type.includes("application/x-ndjson")) { + return true; + } + + return false; + }, + }), + ); + this.koaApp.use(this.makeAssetsServingController()); + this.koaApp.use(this.registry.makeShimController()); + } + + // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context + // TraceId is expected to be request-bound and unique. So these two has to be distinguished. + @runOnce() + override insertAsyncHookMiddleware() { + const asyncHookMiddleware = async ( + ctx: Context, + next: () => Promise, + ) => { + const googleTraceId = ctx.get("x-cloud-trace-context").split("/")?.[0]; + this.threadLocal.setup({ + traceId: randomUUID(), + traceT0: new Date(), + googleTraceId, + }); + + return next(); + }; + + this.koaApp.use(asyncHookMiddleware); + } + + @Finalizer() + override async standDown() { + const tasks: Promise[] = []; + if (this.httpAlternativeServer?.listening) { + (this.httpAlternativeServer as http.Server).closeIdleConnections?.(); + this.httpAlternativeServer.close(); + tasks.push( + new Promise((resolve, reject) => { + this.httpAlternativeServer!.close((err) => { + if (err) { + return reject(err); } - } - - await super.init(); + resolve(); + }); + }), + ); } - - override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { - return super.dependencyReady(timeoutMilliseconds); - } - - async walkForAssets() { - const files = await FsWalk.walkOut(path.resolve(__dirname, '..', '..', 'public')); - - for (const file of files) { - if (file.type !== 'file') { - continue; - } - this.assets.set(file.relativePath.toString(), file); - } - } - - override listen(port: number) { - const r = super.listen(port); - if (this.httpAlternativeServer) { - const altPort = port + 1; - this.httpAlternativeServer.listen(altPort, () => { - this.logger.info(`Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`); - }); - } - - return r; - } - - makeAssetsServingController() { - return (ctx: Context, next: Next) => { - const requestPath = ctx.path; - const file = requestPath.slice(1); - if (!file) { - return next(); - } - - const asset = this.assets.get(file); - if (asset?.type !== 'file') { - return next(); - } - - ctx.body = fs.createReadStream(asset.path); - ctx.type = mimeOfExt(path.extname(asset.path.toString())) || 'application/octet-stream'; - ctx.set('Content-Length', asset.stats.size.toString()); - - return; - }; - } - - registerRoutes(): void { - this.koaApp.use(koaCompress({ - filter(type) { - if (type.startsWith('text/')) { - return true; - } - - if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) { - return true; - } - - if (type.includes('application/x-ndjson')) { - return true; - } - - return false; - } - })); - this.koaApp.use(this.makeAssetsServingController()); - this.koaApp.use(this.registry.makeShimController()); - } - - - // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context - // TraceId is expected to be request-bound and unique. So these two has to be distinguished. - @runOnce() - override insertAsyncHookMiddleware() { - const asyncHookMiddleware = async (ctx: Context, next: () => Promise) => { - const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0]; - this.threadLocal.setup({ - traceId: randomUUID(), - traceT0: new Date(), - googleTraceId, - }); - - return next(); - }; - - this.koaApp.use(asyncHookMiddleware); - } - - @Finalizer() - override async standDown() { - const tasks: Promise[] = []; - if (this.httpAlternativeServer?.listening) { - (this.httpAlternativeServer as http.Server).closeIdleConnections?.(); - this.httpAlternativeServer.close(); - tasks.push(new Promise((resolve, reject) => { - this.httpAlternativeServer!.close((err) => { - if (err) { - return reject(err); - } - resolve(); - }); - })); - } - tasks.push(super.standDown()); - await Promise.all(tasks); - } - + tasks.push(super.standDown()); + await Promise.all(tasks); + } } const instance = container.resolve(SearchStandAloneServer); export default instance; -if (process.env.NODE_ENV?.includes('dry-run')) { - instance.serviceReady().then(() => finalizer.terminate()); +if (process.env.NODE_ENV?.includes("dry-run")) { + instance.serviceReady().then(() => finalizer.terminate()); } else { - instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000)); + instance + .serviceReady() + .then((s) => s.h2c().listen(parseInt(process.env.PORT || "") || 3000)); } diff --git a/src/stand-alone/serp.ts b/src/stand-alone/serp.ts index 94d53b4..d6a76da 100644 --- a/src/stand-alone/serp.ts +++ b/src/stand-alone/serp.ts @@ -1,185 +1,202 @@ -import 'reflect-metadata'; -import { container, singleton } from 'tsyringe'; +import "reflect-metadata"; +import { container, singleton } from "tsyringe"; -import { KoaServer } from 'civkit/civ-rpc/koa'; -import http2 from 'http2'; -import http from 'http'; -import { FsWalk, WalkOutEntity } from 'civkit/fswalk'; -import path from 'path'; -import fs from 'fs'; -import { mimeOfExt } from 'civkit/mime'; -import { Context, Next } from 'koa'; -import { RPCRegistry } from '../services/registry'; -import { AsyncResource } from 'async_hooks'; -import { runOnce } from 'civkit/decorators'; -import { randomUUID } from 'crypto'; -import { ThreadedServiceRegistry } from '../services/threaded'; -import { GlobalLogger } from '../services/logger'; -import { AsyncLocalContext } from '../services/async-context'; -import finalizer, { Finalizer } from '../services/finalizer'; -import { SerpHost } from '../api/serp'; -import koaCompress from 'koa-compress'; -import { getAuditionMiddleware } from '../shared/utils/audition'; -import { STANDALONE_BOOT_TIMEOUT_MS } from '../services/boot-timeouts'; +import { KoaServer } from "civkit/civ-rpc/koa"; +import http2 from "http2"; +import http from "http"; +import { FsWalk, WalkOutEntity } from "civkit/fswalk"; +import path from "path"; +import fs from "fs"; +import { mimeOfExt } from "civkit/mime"; +import { Context, Next } from "koa"; +import { RPCRegistry } from "../services/registry"; +import { AsyncResource } from "async_hooks"; +import { runOnce } from "civkit/decorators"; +import { randomUUID } from "crypto"; +import { ThreadedServiceRegistry } from "../services/threaded"; +import { GlobalLogger } from "../services/logger"; +import { AsyncLocalContext } from "../services/async-context"; +import finalizer, { Finalizer } from "../services/finalizer"; +import { SerpHost } from "../api/serp"; +import koaCompress from "koa-compress"; +import { getAuditionMiddleware } from "../shared/utils/audition"; +import { STANDALONE_BOOT_TIMEOUT_MS } from "../services/boot-timeouts"; @singleton() export class SERPStandAloneServer extends KoaServer { - logger = this.globalLogger.child({ service: this.constructor.name }); + logger = this.globalLogger.child({ service: this.constructor.name }); - httpAlternativeServer?: typeof this['httpServer']; - assets = new Map(); + httpAlternativeServer?: (typeof this)["httpServer"]; + assets = new Map(); - constructor( - protected globalLogger: GlobalLogger, - protected registry: RPCRegistry, - protected serpHost: SerpHost, - protected threadLocal: AsyncLocalContext, - protected threads: ThreadedServiceRegistry, - ) { - super(...arguments); + constructor( + protected globalLogger: GlobalLogger, + protected registry: RPCRegistry, + protected serpHost: SerpHost, + protected threadLocal: AsyncLocalContext, + protected threads: ThreadedServiceRegistry, + ) { + super(...arguments); + } + + h2c() { + this.httpAlternativeServer = this.httpServer; + const fn = this.koaApp.callback(); + this.httpServer = http2.createServer((req, res) => { + const ar = new AsyncResource("HTTP2ServerRequest"); + ar.runInAsyncScope(fn, this.koaApp, req, res); + }); + // useResourceBasedDefaultTracker(); + + return this; + } + + override async init() { + await this.walkForAssets(); + await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + + for (const [k, v] of this.registry.conf.entries()) { + if (v.tags?.includes("crawl")) { + this.registry.conf.delete(k); + } } - h2c() { - this.httpAlternativeServer = this.httpServer; - const fn = this.koaApp.callback(); - this.httpServer = http2.createServer((req, res) => { - const ar = new AsyncResource('HTTP2ServerRequest'); - ar.runInAsyncScope(fn, this.koaApp, req, res); - }); - // useResourceBasedDefaultTracker(); + await super.init(); + } - return this; + override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { + return super.dependencyReady(timeoutMilliseconds); + } + + async walkForAssets() { + const files = await FsWalk.walkOut( + path.resolve(__dirname, "..", "..", "public"), + ); + + for (const file of files) { + if (file.type !== "file") { + continue; + } + this.assets.set(file.relativePath.toString(), file); + } + } + + override listen(port: number) { + const r = super.listen(port); + if (this.httpAlternativeServer) { + const altPort = port + 1; + this.httpAlternativeServer.listen(altPort, () => { + this.logger.info( + `Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`, + ); + }); } - override async init() { - await this.walkForAssets(); - await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS); + return r; + } - for (const [k, v] of this.registry.conf.entries()) { - if (v.tags?.includes('crawl')) { - this.registry.conf.delete(k); + makeAssetsServingController() { + return (ctx: Context, next: Next) => { + const requestPath = ctx.path; + const file = requestPath.slice(1); + if (!file) { + return next(); + } + + const asset = this.assets.get(file); + if (asset?.type !== "file") { + return next(); + } + + ctx.body = fs.createReadStream(asset.path); + ctx.type = + mimeOfExt(path.extname(asset.path.toString())) || + "application/octet-stream"; + ctx.set("Content-Length", asset.stats.size.toString()); + + return; + }; + } + + registerRoutes(): void { + this.koaApp.use(getAuditionMiddleware()); + this.koaApp.use( + koaCompress({ + filter(type) { + if (type.startsWith("text/")) { + return true; + } + + if ( + type.includes("application/json") || + type.includes("+json") || + type.includes("+xml") + ) { + return true; + } + + if (type.includes("application/x-ndjson")) { + return true; + } + + return false; + }, + }), + ); + this.koaApp.use(this.makeAssetsServingController()); + this.koaApp.use(this.registry.makeShimController()); + } + + // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context + // TraceId is expected to be request-bound and unique. So these two has to be distinguished. + @runOnce() + override insertAsyncHookMiddleware() { + const asyncHookMiddleware = async ( + ctx: Context, + next: () => Promise, + ) => { + const googleTraceId = ctx.get("x-cloud-trace-context").split("/")?.[0]; + this.threadLocal.setup({ + traceId: randomUUID(), + traceT0: new Date(), + googleTraceId, + }); + + return next(); + }; + + this.koaApp.use(asyncHookMiddleware); + } + + @Finalizer() + override async standDown() { + const tasks: Promise[] = []; + if (this.httpAlternativeServer?.listening) { + (this.httpAlternativeServer as http.Server).closeIdleConnections?.(); + this.httpAlternativeServer.close(); + tasks.push( + new Promise((resolve, reject) => { + this.httpAlternativeServer!.close((err) => { + if (err) { + return reject(err); } - } - - await super.init(); + resolve(); + }); + }), + ); } - - override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) { - return super.dependencyReady(timeoutMilliseconds); - } - - async walkForAssets() { - const files = await FsWalk.walkOut(path.resolve(__dirname, '..', '..', 'public')); - - for (const file of files) { - if (file.type !== 'file') { - continue; - } - this.assets.set(file.relativePath.toString(), file); - } - } - - override listen(port: number) { - const r = super.listen(port); - if (this.httpAlternativeServer) { - const altPort = port + 1; - this.httpAlternativeServer.listen(altPort, () => { - this.logger.info(`Alternative ${this.httpAlternativeServer!.constructor.name} listening on port ${altPort}`); - }); - } - - return r; - } - - makeAssetsServingController() { - return (ctx: Context, next: Next) => { - const requestPath = ctx.path; - const file = requestPath.slice(1); - if (!file) { - return next(); - } - - const asset = this.assets.get(file); - if (asset?.type !== 'file') { - return next(); - } - - ctx.body = fs.createReadStream(asset.path); - ctx.type = mimeOfExt(path.extname(asset.path.toString())) || 'application/octet-stream'; - ctx.set('Content-Length', asset.stats.size.toString()); - - return; - }; - } - - registerRoutes(): void { - this.koaApp.use(getAuditionMiddleware()); - this.koaApp.use(koaCompress({ - filter(type) { - if (type.startsWith('text/')) { - return true; - } - - if (type.includes('application/json') || type.includes('+json') || type.includes('+xml')) { - return true; - } - - if (type.includes('application/x-ndjson')) { - return true; - } - - return false; - } - })); - this.koaApp.use(this.makeAssetsServingController()); - this.koaApp.use(this.registry.makeShimController()); - } - - - // Using h2c server has an implication that multiple requests may share the same connection and x-cloud-trace-context - // TraceId is expected to be request-bound and unique. So these two has to be distinguished. - @runOnce() - override insertAsyncHookMiddleware() { - const asyncHookMiddleware = async (ctx: Context, next: () => Promise) => { - const googleTraceId = ctx.get('x-cloud-trace-context').split('/')?.[0]; - this.threadLocal.setup({ - traceId: randomUUID(), - traceT0: new Date(), - googleTraceId, - }); - - return next(); - }; - - this.koaApp.use(asyncHookMiddleware); - } - - @Finalizer() - override async standDown() { - const tasks: Promise[] = []; - if (this.httpAlternativeServer?.listening) { - (this.httpAlternativeServer as http.Server).closeIdleConnections?.(); - this.httpAlternativeServer.close(); - tasks.push(new Promise((resolve, reject) => { - this.httpAlternativeServer!.close((err) => { - if (err) { - return reject(err); - } - resolve(); - }); - })); - } - tasks.push(super.standDown()); - await Promise.all(tasks); - } - + tasks.push(super.standDown()); + await Promise.all(tasks); + } } const instance = container.resolve(SERPStandAloneServer); export default instance; -if (process.env.NODE_ENV?.includes('dry-run')) { - instance.serviceReady().then(() => finalizer.terminate()); +if (process.env.NODE_ENV?.includes("dry-run")) { + instance.serviceReady().then(() => finalizer.terminate()); } else { - instance.serviceReady().then((s) => s.h2c().listen(parseInt(process.env.PORT || '') || 3000)); + instance + .serviceReady() + .then((s) => s.h2c().listen(parseInt(process.env.PORT || "") || 3000)); } diff --git a/src/types.d.ts b/src/types.d.ts index b2ab9ce..cbe7792 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,28 +1,28 @@ -declare module 'langdetect' { - interface DetectionResult { - lang: string; - prob: number; - } +declare module "langdetect" { + interface DetectionResult { + lang: string; + prob: number; + } - export function detect(text: string): DetectionResult[]; - export function detectOne(text: string): string | null; + export function detect(text: string): DetectionResult[]; + export function detectOne(text: string): string | null; } -declare module 'jsdom' { - import EventEmitter from 'events'; - export class JSDOM { - constructor(html: string, options?: any); - window: typeof window; - } - export class VirtualConsole extends EventEmitter { - constructor(); - sendTo(console: any, options?: any); - } +declare module "jsdom" { + import EventEmitter from "events"; + export class JSDOM { + constructor(html: string, options?: any); + window: typeof window; + } + export class VirtualConsole extends EventEmitter { + constructor(); + sendTo(console: any, options?: any); + } } -declare module 'simple-zstd' { - import { Duplex } from 'stream'; - export function ZSTDCompress(lvl: Number): Duplex; - export function ZSTDDecompress(): Duplex; - export function ZSTDDecompressMaybe(): Duplex; +declare module "simple-zstd" { + import { Duplex } from "stream"; + export function ZSTDCompress(lvl: Number): Duplex; + export function ZSTDDecompress(): Duplex; + export function ZSTDDecompressMaybe(): Duplex; } diff --git a/src/utils/encoding.ts b/src/utils/encoding.ts index 0f34a9b..271408a 100644 --- a/src/utils/encoding.ts +++ b/src/utils/encoding.ts @@ -1,34 +1,39 @@ -import { createReadStream } from 'fs'; -import { Readable } from 'stream'; -import { TextDecoderStream } from 'stream/web'; +import { createReadStream } from "fs"; +import { Readable } from "stream"; +import { TextDecoderStream } from "stream/web"; export async function decodeFileStream( - fileStream: Readable, - encoding: string = 'utf-8', + fileStream: Readable, + encoding: string = "utf-8", ): Promise { - const decodeStream = new TextDecoderStream(encoding, { fatal: false, ignoreBOM: false }); - Readable.toWeb(fileStream).pipeThrough(decodeStream); - const chunks = []; + const decodeStream = new TextDecoderStream(encoding, { + fatal: false, + ignoreBOM: false, + }); + Readable.toWeb(fileStream).pipeThrough(decodeStream); + const chunks = []; - for await (const chunk of decodeStream.readable) { - chunks.push(chunk); - } + for await (const chunk of decodeStream.readable) { + chunks.push(chunk); + } - return chunks.join(''); + return chunks.join(""); } - export async function readFile( - filePath: string, - encoding: string = 'utf-8', + filePath: string, + encoding: string = "utf-8", ): Promise { - const decodeStream = new TextDecoderStream(encoding, { fatal: false, ignoreBOM: false }); - Readable.toWeb(createReadStream(filePath)).pipeThrough(decodeStream); - const chunks = []; + const decodeStream = new TextDecoderStream(encoding, { + fatal: false, + ignoreBOM: false, + }); + Readable.toWeb(createReadStream(filePath)).pipeThrough(decodeStream); + const chunks = []; - for await (const chunk of decodeStream.readable) { - chunks.push(chunk); - } + for await (const chunk of decodeStream.readable) { + chunks.push(chunk); + } - return chunks.join(''); -} \ No newline at end of file + return chunks.join(""); +} diff --git a/src/utils/ip.ts b/src/utils/ip.ts index d8d387e..fb92891 100644 --- a/src/utils/ip.ts +++ b/src/utils/ip.ts @@ -1,167 +1,167 @@ -import { isIPv4, isIPv6 } from 'net'; +import { isIPv4, isIPv6 } from "net"; export function parseIp(ip: string): Buffer { - if (isIPv4(ip)) { - const [a, b, c, d] = ip.split('.').map(Number); + if (isIPv4(ip)) { + const [a, b, c, d] = ip.split(".").map(Number); - const buf = Buffer.alloc(4); - buf.writeUInt8(a, 0); - buf.writeUInt8(b, 1); - buf.writeUInt8(c, 2); - buf.writeUInt8(d, 3); + const buf = Buffer.alloc(4); + buf.writeUInt8(a, 0); + buf.writeUInt8(b, 1); + buf.writeUInt8(c, 2); + buf.writeUInt8(d, 3); - return buf; + return buf; + } + + if (isIPv6(ip)) { + if (ip.includes(".")) { + const parts = ip.split(":"); + const ipv4Part = parts.pop(); + if (!ipv4Part) throw new Error("Invalid IPv6 address"); + const ipv4Bytes = parseIp(ipv4Part); + parts.push("0"); + const ipv6Bytes = parseIp(parts.join(":")); + ipv6Bytes.writeUInt32BE(ipv4Bytes.readUInt32BE(0), 12); + + return ipv6Bytes; } - if (isIPv6(ip)) { - if (ip.includes('.')) { - const parts = ip.split(':'); - const ipv4Part = parts.pop(); - if (!ipv4Part) throw new Error('Invalid IPv6 address'); - const ipv4Bytes = parseIp(ipv4Part); - parts.push('0'); - const ipv6Bytes = parseIp(parts.join(':')); - ipv6Bytes.writeUInt32BE(ipv4Bytes.readUInt32BE(0), 12); + const buf = Buffer.alloc(16); - return ipv6Bytes; - } - - const buf = Buffer.alloc(16); - - // Expand :: notation - let expanded = ip; - if (ip.includes('::')) { - const sides = ip.split('::'); - const left = sides[0] ? sides[0].split(':') : []; - const right = sides[1] ? sides[1].split(':') : []; - const middle = Array(8 - left.length - right.length).fill('0'); - expanded = [...left, ...middle, ...right].join(':'); - } - - // Convert to buffer - const parts = expanded.split(':'); - let offset = 0; - for (const part of parts) { - buf.writeUInt16BE(parseInt(part, 16), offset); - offset += 2; - } - - return buf; + // Expand :: notation + let expanded = ip; + if (ip.includes("::")) { + const sides = ip.split("::"); + const left = sides[0] ? sides[0].split(":") : []; + const right = sides[1] ? sides[1].split(":") : []; + const middle = Array(8 - left.length - right.length).fill("0"); + expanded = [...left, ...middle, ...right].join(":"); } - throw new Error('Invalid IP address'); + // Convert to buffer + const parts = expanded.split(":"); + let offset = 0; + for (const part of parts) { + buf.writeUInt16BE(parseInt(part, 16), offset); + offset += 2; + } + + return buf; + } + + throw new Error("Invalid IP address"); } - export function parseCIDR(cidr: string): [Buffer, Buffer] { - const [ip, prefixTxt] = cidr.split('/'); - const buf = parseIp(ip); - const maskBuf = Buffer.alloc(buf.byteLength, 0xff); - const prefixBits = parseInt(prefixTxt); + const [ip, prefixTxt] = cidr.split("/"); + const buf = parseIp(ip); + const maskBuf = Buffer.alloc(buf.byteLength, 0xff); + const prefixBits = parseInt(prefixTxt); - let offsetBits = 0; - while (offsetBits < (buf.byteLength * 8)) { - if (offsetBits <= (prefixBits - 8)) { - offsetBits += 8; - continue; - } - const bitsRemain = prefixBits - offsetBits; - const byteOffset = Math.floor(offsetBits / 8); - - if (bitsRemain > 0) { - const theByte = buf[byteOffset]; - const mask = 0xff << (8 - bitsRemain); - maskBuf[byteOffset] = mask; - buf[byteOffset] = theByte & mask; - - offsetBits += 8; - continue; - }; - buf[byteOffset] = 0; - maskBuf[byteOffset] = 0; - - offsetBits += 8; + let offsetBits = 0; + while (offsetBits < buf.byteLength * 8) { + if (offsetBits <= prefixBits - 8) { + offsetBits += 8; + continue; } + const bitsRemain = prefixBits - offsetBits; + const byteOffset = Math.floor(offsetBits / 8); - return [buf, maskBuf]; + if (bitsRemain > 0) { + const theByte = buf[byteOffset]; + const mask = 0xff << (8 - bitsRemain); + maskBuf[byteOffset] = mask; + buf[byteOffset] = theByte & mask; + + offsetBits += 8; + continue; + } + buf[byteOffset] = 0; + maskBuf[byteOffset] = 0; + + offsetBits += 8; + } + + return [buf, maskBuf]; } export class CIDR { - buff: Buffer; - mask: Buffer; - text: string; - constructor(cidr: string) { - this.text = cidr; - [this.buff, this.mask] = parseCIDR(cidr); + buff: Buffer; + mask: Buffer; + text: string; + constructor(cidr: string) { + this.text = cidr; + [this.buff, this.mask] = parseCIDR(cidr); + } + + toString() { + return this.text; + } + + get family() { + return this.buff.byteLength === 4 ? 4 : 6; + } + + test(ip: string | Buffer): boolean { + const parsedIp = typeof ip === "string" ? parseIp(ip) : ip; + + if (parsedIp.byteLength !== this.buff.byteLength) { + return false; } - toString() { - return this.text; - } - - get family() { - return this.buff.byteLength === 4 ? 4 : 6; - } - - test(ip: string | Buffer): boolean { - const parsedIp = typeof ip === 'string' ? parseIp(ip) : ip; - - if (parsedIp.byteLength !== this.buff.byteLength) { - return false; - } - - for (const i of Array(this.buff.byteLength).keys()) { - const t = parsedIp[i]; - const m = this.mask[i]; - - if (m === 0) { - return true; - } - - const r = this.buff[i]; - if ((t & m) !== r) { - return false; - } - } + for (const i of Array(this.buff.byteLength).keys()) { + const t = parsedIp[i]; + const m = this.mask[i]; + if (m === 0) { return true; + } + + const r = this.buff[i]; + if ((t & m) !== r) { + return false; + } } + + return true; + } } const nonPublicNetworks4 = [ - '10.0.0.0/8', - '172.16.0.0/12', - '192.168.0.0/16', + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", - '127.0.0.0/8', - '255.255.255.255/32', - '169.254.0.0/16', - '224.0.0.0/4', + "127.0.0.0/8", + "255.255.255.255/32", + "169.254.0.0/16", + "224.0.0.0/4", - '100.64.0.0/10', - '0.0.0.0/32', + "100.64.0.0/10", + "0.0.0.0/32", ]; - const nonPublicNetworks6 = [ - 'fc00::/7', - 'fe80::/10', - 'ff00::/8', + "fc00::/7", + "fe80::/10", + "ff00::/8", - '::127.0.0.0/104', - '::/128', + "::127.0.0.0/104", + "::/128", ]; -const nonPublicCIDRs = [...nonPublicNetworks4, ...nonPublicNetworks6].map(cidr => new CIDR(cidr)); +const nonPublicCIDRs = [...nonPublicNetworks4, ...nonPublicNetworks6].map( + (cidr) => new CIDR(cidr), +); export function isIPInNonPublicRange(ip: string) { - const parsed = parseIp(ip); + const parsed = parseIp(ip); - for (const cidr of nonPublicCIDRs) { - if (cidr.test(parsed)) { - return true; - } + for (const cidr of nonPublicCIDRs) { + if (cidr.test(parsed)) { + return true; } + } - return false; + return false; } diff --git a/src/utils/markdown.ts b/src/utils/markdown.ts index f8def89..1b99ff8 100644 --- a/src/utils/markdown.ts +++ b/src/utils/markdown.ts @@ -1,39 +1,46 @@ - export function tidyMarkdown(markdown: string): string { + // Step 1: Handle complex broken links with text and optional images spread across multiple lines + let normalizedMarkdown = markdown.replace( + /\[\s*([^\]\n]+?)\s*\]\s*\(\s*([^)]+)\s*\)/g, + (match, text, url) => { + // Remove internal new lines and excessive spaces within the text + text = text.replace(/\s+/g, " ").trim(); + url = url.replace(/\s+/g, "").trim(); + return `[${text}](${url})`; + }, + ); - // Step 1: Handle complex broken links with text and optional images spread across multiple lines - let normalizedMarkdown = markdown.replace(/\[\s*([^\]\n]+?)\s*\]\s*\(\s*([^)]+)\s*\)/g, (match, text, url) => { - // Remove internal new lines and excessive spaces within the text - text = text.replace(/\s+/g, ' ').trim(); - url = url.replace(/\s+/g, '').trim(); - return `[${text}](${url})`; - }); + normalizedMarkdown = normalizedMarkdown.replace( + /\[\s*([^\]\n!]*?)\s*\n*(?:!\[([^\]]*)\]\((.*?)\))?\s*\n*\]\s*\(\s*([^)]+)\s*\)/g, + (match, text, alt, imgUrl, linkUrl) => { + // Normalize by removing excessive spaces and new lines + text = text.replace(/\s+/g, " ").trim(); + alt = alt ? alt.replace(/\s+/g, " ").trim() : ""; + imgUrl = imgUrl ? imgUrl.replace(/\s+/g, "").trim() : ""; + linkUrl = linkUrl.replace(/\s+/g, "").trim(); + if (imgUrl) { + return `[${text} ![${alt}](${imgUrl})](${linkUrl})`; + } else { + return `[${text}](${linkUrl})`; + } + }, + ); - normalizedMarkdown = normalizedMarkdown.replace(/\[\s*([^\]\n!]*?)\s*\n*(?:!\[([^\]]*)\]\((.*?)\))?\s*\n*\]\s*\(\s*([^)]+)\s*\)/g, (match, text, alt, imgUrl, linkUrl) => { - // Normalize by removing excessive spaces and new lines - text = text.replace(/\s+/g, ' ').trim(); - alt = alt ? alt.replace(/\s+/g, ' ').trim() : ''; - imgUrl = imgUrl ? imgUrl.replace(/\s+/g, '').trim() : ''; - linkUrl = linkUrl.replace(/\s+/g, '').trim(); - if (imgUrl) { - return `[${text} ![${alt}](${imgUrl})](${linkUrl})`; - } else { - return `[${text}](${linkUrl})`; - } - }); + // Step 2: Normalize regular links that may be broken across lines + normalizedMarkdown = normalizedMarkdown.replace( + /\[\s*([^\]]+)\]\s*\(\s*([^)]+)\)/g, + (match, text, url) => { + text = text.replace(/\s+/g, " ").trim(); + url = url.replace(/\s+/g, "").trim(); + return `[${text}](${url})`; + }, + ); - // Step 2: Normalize regular links that may be broken across lines - normalizedMarkdown = normalizedMarkdown.replace(/\[\s*([^\]]+)\]\s*\(\s*([^)]+)\)/g, (match, text, url) => { - text = text.replace(/\s+/g, ' ').trim(); - url = url.replace(/\s+/g, '').trim(); - return `[${text}](${url})`; - }); + // Step 3: Replace more than two consecutive empty lines with exactly two empty lines + normalizedMarkdown = normalizedMarkdown.replace(/\n{3,}/g, "\n\n"); - // Step 3: Replace more than two consecutive empty lines with exactly two empty lines - normalizedMarkdown = normalizedMarkdown.replace(/\n{3,}/g, '\n\n'); + // Step 4: Remove leading spaces from each line + normalizedMarkdown = normalizedMarkdown.replace(/^[ \t]+/gm, ""); - // Step 4: Remove leading spaces from each line - normalizedMarkdown = normalizedMarkdown.replace(/^[ \t]+/gm, ''); - - return normalizedMarkdown.trim(); + return normalizedMarkdown.trim(); } diff --git a/src/utils/misc.ts b/src/utils/misc.ts index 71bfc2d..6cc37c0 100644 --- a/src/utils/misc.ts +++ b/src/utils/misc.ts @@ -1,27 +1,25 @@ -import { ParamValidationError } from 'civkit'; +import { ParamValidationError } from "civkit"; export function cleanAttribute(attribute: string | null) { - return attribute ? attribute.replace(/(\n+\s*)+/g, '\n') : ''; + return attribute ? attribute.replace(/(\n+\s*)+/g, "\n") : ""; } - export function tryDecodeURIComponent(input: string) { - try { - return decodeURIComponent(input); - } catch (_err) { - if (URL.canParse(input, 'http://localhost:3000')) { - return input; - } - - throw new ParamValidationError(`Invalid URIComponent: ${input}`); + try { + return decodeURIComponent(input); + } catch (_err) { + if (URL.canParse(input, "http://localhost:3000")) { + return input; } + + throw new ParamValidationError(`Invalid URIComponent: ${input}`); + } } - export async function* toAsyncGenerator(val: T) { - yield val; + yield val; } export async function* toGenerator(val: T) { - yield val; + yield val; } diff --git a/src/utils/tailwind-classes.ts b/src/utils/tailwind-classes.ts index 7f82d00..5ab9d0d 100644 --- a/src/utils/tailwind-classes.ts +++ b/src/utils/tailwind-classes.ts @@ -1,9625 +1,9625 @@ export const tailwindClasses = new Set([ - "aspect-auto", - "aspect-square", - "aspect-video", - "container", - "columns-1", - "columns-2", - "columns-3", - "columns-4", - "columns-5", - "columns-6", - "columns-7", - "columns-8", - "columns-9", - "columns-10", - "columns-11", - "columns-12", - "columns-auto", - "columns-3xs", - "columns-2xs", - "columns-xs", - "columns-sm", - "columns-md", - "columns-lg", - "columns-xl", - "columns-2xl", - "columns-3xl", - "columns-4xl", - "columns-5xl", - "columns-6xl", - "columns-7xl", - "break-after-auto", - "break-after-avoid", - "break-after-all", - "break-after-avoid-page", - "break-after-page", - "break-after-left", - "break-after-right", - "break-after-column", - "break-before-auto", - "break-before-avoid", - "break-before-all", - "break-before-avoid-page", - "break-before-page", - "break-before-left", - "break-before-right", - "break-before-column", - "break-inside-auto", - "break-inside-avoid", - "break-inside-avoid-page", - "break-inside-avoid-column", - "box-decoration-clone", - "box-decoration-slice", - "box-border", - "box-content", - "block", - "inline-block", - "inline", - "flex", - "inline-flex", - "table", - "inline-table", - "table-caption", - "table-cell", - "table-column", - "table-column-group", - "table-footer-group", - "table-header-group", - "table-row-group", - "table-row", - "flow-root", - "grid", - "inline-grid", - "contents", - "list-item", - // "hidden", - "float-start", - "float-end", - "float-right", - "float-left", - "float-none", - "clear-start", - "clear-end", - "clear-left", - "clear-right", - "clear-both", - "clear-none", - "isolate", - "isolation-auto", - "object-contain", - "object-cover", - "object-fill", - "object-none", - "object-scale-down", - "object-bottom", - "object-center", - "object-left", - "object-left-bottom", - "object-left-top", - "object-right", - "object-right-bottom", - "object-right-top", - "object-top", - "overflow-auto", - "overflow-hidden", - "overflow-clip", - "overflow-visible", - "overflow-scroll", - "overflow-x-auto", - "overflow-y-auto", - "overflow-x-hidden", - "overflow-y-hidden", - "overflow-x-clip", - "overflow-y-clip", - "overflow-x-visible", - "overflow-y-visible", - "overflow-x-scroll", - "overflow-y-scroll", - "overscroll-auto", - "overscroll-contain", - "overscroll-none", - "overscroll-y-auto", - "overscroll-y-contain", - "overscroll-y-none", - "overscroll-x-auto", - "overscroll-x-contain", - "overscroll-x-none", - "static", - "fixed", - "absolute", - "relative", - "sticky", - "inset-0", - "inset-x-0", - "inset-y-0", - "start-0", - "end-0", - "top-0", - "right-0", - "bottom-0", - "left-0", - "inset-px", - "inset-x-px", - "inset-y-px", - "start-px", - "end-px", - "top-px", - "right-px", - "bottom-px", - "left-px", - "inset-0.5", - "inset-x-0.5", - "inset-y-0.5", - "start-0.5", - "end-0.5", - "top-0.5", - "right-0.5", - "bottom-0.5", - "left-0.5", - "inset-1", - "inset-x-1", - "inset-y-1", - "start-1", - "end-1", - "top-1", - "right-1", - "bottom-1", - "left-1", - "inset-1.5", - "inset-x-1.5", - "inset-y-1.5", - "start-1.5", - "end-1.5", - "top-1.5", - "right-1.5", - "bottom-1.5", - "left-1.5", - "inset-2", - "inset-x-2", - "inset-y-2", - "start-2", - "end-2", - "top-2", - "right-2", - "bottom-2", - "left-2", - "inset-2.5", - "inset-x-2.5", - "inset-y-2.5", - "start-2.5", - "end-2.5", - "top-2.5", - "right-2.5", - "bottom-2.5", - "left-2.5", - "inset-3", - "inset-x-3", - "inset-y-3", - "start-3", - "end-3", - "top-3", - "right-3", - "bottom-3", - "left-3", - "inset-3.5", - "inset-x-3.5", - "inset-y-3.5", - "start-3.5", - "end-3.5", - "top-3.5", - "right-3.5", - "bottom-3.5", - "left-3.5", - "inset-4", - "inset-x-4", - "inset-y-4", - "start-4", - "end-4", - "top-4", - "right-4", - "bottom-4", - "left-4", - "inset-5", - "inset-x-5", - "inset-y-5", - "start-5", - "end-5", - "top-5", - "right-5", - "bottom-5", - "left-5", - "inset-6", - "inset-x-6", - "inset-y-6", - "start-6", - "end-6", - "top-6", - "right-6", - "bottom-6", - "left-6", - "inset-7", - "inset-x-7", - "inset-y-7", - "start-7", - "end-7", - "top-7", - "right-7", - "bottom-7", - "left-7", - "inset-8", - "inset-x-8", - "inset-y-8", - "start-8", - "end-8", - "top-8", - "right-8", - "bottom-8", - "left-8", - "inset-9", - "inset-x-9", - "inset-y-9", - "start-9", - "end-9", - "top-9", - "right-9", - "bottom-9", - "left-9", - "inset-10", - "inset-x-10", - "inset-y-10", - "start-10", - "end-10", - "top-10", - "right-10", - "bottom-10", - "left-10", - "inset-11", - "inset-x-11", - "inset-y-11", - "start-11", - "end-11", - "top-11", - "right-11", - "bottom-11", - "left-11", - "inset-12", - "inset-x-12", - "inset-y-12", - "start-12", - "end-12", - "top-12", - "right-12", - "bottom-12", - "left-12", - "inset-14", - "inset-x-14", - "inset-y-14", - "start-14", - "end-14", - "top-14", - "right-14", - "bottom-14", - "left-14", - "inset-16", - "inset-x-16", - "inset-y-16", - "start-16", - "end-16", - "top-16", - "right-16", - "bottom-16", - "left-16", - "inset-20", - "inset-x-20", - "inset-y-20", - "start-20", - "end-20", - "top-20", - "right-20", - "bottom-20", - "left-20", - "inset-24", - "inset-x-24", - "inset-y-24", - "start-24", - "end-24", - "top-24", - "right-24", - "bottom-24", - "left-24", - "inset-28", - "inset-x-28", - "inset-y-28", - "start-28", - "end-28", - "top-28", - "right-28", - "bottom-28", - "left-28", - "inset-32", - "inset-x-32", - "inset-y-32", - "start-32", - "end-32", - "top-32", - "right-32", - "bottom-32", - "left-32", - "inset-36", - "inset-x-36", - "inset-y-36", - "start-36", - "end-36", - "top-36", - "right-36", - "bottom-36", - "left-36", - "inset-40", - "inset-x-40", - "inset-y-40", - "start-40", - "end-40", - "top-40", - "right-40", - "bottom-40", - "left-40", - "inset-44", - "inset-x-44", - "inset-y-44", - "start-44", - "end-44", - "top-44", - "right-44", - "bottom-44", - "left-44", - "inset-48", - "inset-x-48", - "inset-y-48", - "start-48", - "end-48", - "top-48", - "right-48", - "bottom-48", - "left-48", - "inset-52", - "inset-x-52", - "inset-y-52", - "start-52", - "end-52", - "top-52", - "right-52", - "bottom-52", - "left-52", - "inset-56", - "inset-x-56", - "inset-y-56", - "start-56", - "end-56", - "top-56", - "right-56", - "bottom-56", - "left-56", - "inset-60", - "inset-x-60", - "inset-y-60", - "start-60", - "end-60", - "top-60", - "right-60", - "bottom-60", - "left-60", - "inset-64", - "inset-x-64", - "inset-y-64", - "start-64", - "end-64", - "top-64", - "right-64", - "bottom-64", - "left-64", - "inset-72", - "inset-x-72", - "inset-y-72", - "start-72", - "end-72", - "top-72", - "right-72", - "bottom-72", - "left-72", - "inset-80", - "inset-x-80", - "inset-y-80", - "start-80", - "end-80", - "top-80", - "right-80", - "bottom-80", - "left-80", - "inset-96", - "inset-x-96", - "inset-y-96", - "start-96", - "end-96", - "top-96", - "right-96", - "bottom-96", - "left-96", - "inset-auto", - "inset-1/2", - "inset-1/3", - "inset-2/3", - "inset-1/4", - "inset-2/4", - "inset-3/4", - "inset-full", - "inset-x-auto", - "inset-x-1/2", - "inset-x-1/3", - "inset-x-2/3", - "inset-x-1/4", - "inset-x-2/4", - "inset-x-3/4", - "inset-x-full", - "inset-y-auto", - "inset-y-1/2", - "inset-y-1/3", - "inset-y-2/3", - "inset-y-1/4", - "inset-y-2/4", - "inset-y-3/4", - "inset-y-full", - "start-auto", - "start-1/2", - "start-1/3", - "start-2/3", - "start-1/4", - "start-2/4", - "start-3/4", - "start-full", - "end-auto", - "end-1/2", - "end-1/3", - "end-2/3", - "end-1/4", - "end-2/4", - "end-3/4", - "end-full", - "top-auto", - "top-1/2", - "top-1/3", - "top-2/3", - "top-1/4", - "top-2/4", - "top-3/4", - "top-full", - "right-auto", - "right-1/2", - "right-1/3", - "right-2/3", - "right-1/4", - "right-2/4", - "right-3/4", - "right-full", - "bottom-auto", - "bottom-1/2", - "bottom-1/3", - "bottom-2/3", - "bottom-1/4", - "bottom-2/4", - "bottom-3/4", - "bottom-full", - "left-auto", - "left-1/2", - "left-1/3", - "left-2/3", - "left-1/4", - "left-2/4", - "left-3/4", - "left-full", - "visible", - "invisible", - "collapse", - "z-0", - "z-10", - "z-20", - "z-30", - "z-40", - "z-50", - "z-auto", - "basis-0", - "basis-1", - "basis-2", - "basis-3", - "basis-4", - "basis-5", - "basis-6", - "basis-7", - "basis-8", - "basis-9", - "basis-10", - "basis-11", - "basis-12", - "basis-14", - "basis-16", - "basis-20", - "basis-24", - "basis-28", - "basis-32", - "basis-36", - "basis-40", - "basis-44", - "basis-48", - "basis-52", - "basis-56", - "basis-60", - "basis-64", - "basis-72", - "basis-80", - "basis-96", - "basis-auto", - "basis-px", - "basis-0.5", - "basis-1.5", - "basis-2.5", - "basis-3.5", - "basis-1/2", - "basis-1/3", - "basis-2/3", - "basis-1/4", - "basis-2/4", - "basis-3/4", - "basis-1/5", - "basis-2/5", - "basis-3/5", - "basis-4/5", - "basis-1/6", - "basis-2/6", - "basis-3/6", - "basis-4/6", - "basis-5/6", - "basis-1/12", - "basis-2/12", - "basis-3/12", - "basis-4/12", - "basis-5/12", - "basis-6/12", - "basis-7/12", - "basis-8/12", - "basis-9/12", - "basis-10/12", - "basis-11/12", - "basis-full", - "flex-row", - "flex-row-reverse", - "flex-col", - "flex-col-reverse", - "flex-wrap", - "flex-wrap-reverse", - "flex-nowrap", - "flex-1", - "flex-auto", - "flex-initial", - "flex-none", - "grow", - "grow-0", - "shrink", - "shrink-0", - "order-1", - "order-2", - "order-3", - "order-4", - "order-5", - "order-6", - "order-7", - "order-8", - "order-9", - "order-10", - "order-11", - "order-12", - "order-first", - "order-last", - "order-none", - "grid-cols-1", - "grid-cols-2", - "grid-cols-3", - "grid-cols-4", - "grid-cols-5", - "grid-cols-6", - "grid-cols-7", - "grid-cols-8", - "grid-cols-9", - "grid-cols-10", - "grid-cols-11", - "grid-cols-12", - "grid-cols-none", - "grid-cols-subgrid", - "col-auto", - "col-span-1", - "col-span-2", - "col-span-3", - "col-span-4", - "col-span-5", - "col-span-6", - "col-span-7", - "col-span-8", - "col-span-9", - "col-span-10", - "col-span-11", - "col-span-12", - "col-span-full", - "col-start-1", - "col-start-2", - "col-start-3", - "col-start-4", - "col-start-5", - "col-start-6", - "col-start-7", - "col-start-8", - "col-start-9", - "col-start-10", - "col-start-11", - "col-start-12", - "col-start-13", - "col-start-auto", - "col-end-1", - "col-end-2", - "col-end-3", - "col-end-4", - "col-end-5", - "col-end-6", - "col-end-7", - "col-end-8", - "col-end-9", - "col-end-10", - "col-end-11", - "col-end-12", - "col-end-13", - "col-end-auto", - "grid-rows-1", - "grid-rows-2", - "grid-rows-3", - "grid-rows-4", - "grid-rows-5", - "grid-rows-6", - "grid-rows-7", - "grid-rows-8", - "grid-rows-9", - "grid-rows-10", - "grid-rows-11", - "grid-rows-12", - "grid-rows-none", - "grid-rows-subgrid", - "row-auto", - "row-span-1", - "row-span-2", - "row-span-3", - "row-span-4", - "row-span-5", - "row-span-6", - "row-span-7", - "row-span-8", - "row-span-9", - "row-span-10", - "row-span-11", - "row-span-12", - "row-span-full", - "row-start-1", - "row-start-2", - "row-start-3", - "row-start-4", - "row-start-5", - "row-start-6", - "row-start-7", - "row-start-8", - "row-start-9", - "row-start-10", - "row-start-11", - "row-start-12", - "row-start-13", - "row-start-auto", - "row-end-1", - "row-end-2", - "row-end-3", - "row-end-4", - "row-end-5", - "row-end-6", - "row-end-7", - "row-end-8", - "row-end-9", - "row-end-10", - "row-end-11", - "row-end-12", - "row-end-13", - "row-end-auto", - "grid-flow-row", - "grid-flow-col", - "grid-flow-dense", - "grid-flow-row-dense", - "grid-flow-col-dense", - "auto-cols-auto", - "auto-cols-min", - "auto-cols-max", - "auto-cols-fr", - "auto-rows-auto", - "auto-rows-min", - "auto-rows-max", - "auto-rows-fr", - "gap-0", - "gap-x-0", - "gap-y-0", - "gap-px", - "gap-x-px", - "gap-y-px", - "gap-0.5", - "gap-x-0.5", - "gap-y-0.5", - "gap-1", - "gap-x-1", - "gap-y-1", - "gap-1.5", - "gap-x-1.5", - "gap-y-1.5", - "gap-2", - "gap-x-2", - "gap-y-2", - "gap-2.5", - "gap-x-2.5", - "gap-y-2.5", - "gap-3", - "gap-x-3", - "gap-y-3", - "gap-3.5", - "gap-x-3.5", - "gap-y-3.5", - "gap-4", - "gap-x-4", - "gap-y-4", - "gap-5", - "gap-x-5", - "gap-y-5", - "gap-6", - "gap-x-6", - "gap-y-6", - "gap-7", - "gap-x-7", - "gap-y-7", - "gap-8", - "gap-x-8", - "gap-y-8", - "gap-9", - "gap-x-9", - "gap-y-9", - "gap-10", - "gap-x-10", - "gap-y-10", - "gap-11", - "gap-x-11", - "gap-y-11", - "gap-12", - "gap-x-12", - "gap-y-12", - "gap-14", - "gap-x-14", - "gap-y-14", - "gap-16", - "gap-x-16", - "gap-y-16", - "gap-20", - "gap-x-20", - "gap-y-20", - "gap-24", - "gap-x-24", - "gap-y-24", - "gap-28", - "gap-x-28", - "gap-y-28", - "gap-32", - "gap-x-32", - "gap-y-32", - "gap-36", - "gap-x-36", - "gap-y-36", - "gap-40", - "gap-x-40", - "gap-y-40", - "gap-44", - "gap-x-44", - "gap-y-44", - "gap-48", - "gap-x-48", - "gap-y-48", - "gap-52", - "gap-x-52", - "gap-y-52", - "gap-56", - "gap-x-56", - "gap-y-56", - "gap-60", - "gap-x-60", - "gap-y-60", - "gap-64", - "gap-x-64", - "gap-y-64", - "gap-72", - "gap-x-72", - "gap-y-72", - "gap-80", - "gap-x-80", - "gap-y-80", - "gap-96", - "gap-x-96", - "gap-y-96", - "justify-normal", - "justify-start", - "justify-end", - "justify-center", - "justify-between", - "justify-around", - "justify-evenly", - "justify-stretch", - "justify-items-start", - "justify-items-end", - "justify-items-center", - "justify-items-stretch", - "justify-self-auto", - "justify-self-start", - "justify-self-end", - "justify-self-center", - "justify-self-stretch", - "content-normal", - "content-center", - "content-start", - "content-end", - "content-between", - "content-around", - "content-evenly", - "content-baseline", - "content-stretch", - "items-start", - "items-end", - "items-center", - "items-baseline", - "items-stretch", - "self-auto", - "self-start", - "self-end", - "self-center", - "self-stretch", - "self-baseline", - "place-content-center", - "place-content-start", - "place-content-end", - "place-content-between", - "place-content-around", - "place-content-evenly", - "place-content-baseline", - "place-content-stretch", - "place-items-start", - "place-items-end", - "place-items-center", - "place-items-baseline", - "place-items-stretch", - "place-self-auto", - "place-self-start", - "place-self-end", - "place-self-center", - "place-self-stretch", - "p-0", - "px-0", - "py-0", - "ps-0", - "pe-0", - "pt-0", - "pr-0", - "pb-0", - "pl-0", - "p-px", - "px-px", - "py-px", - "ps-px", - "pe-px", - "pt-px", - "pr-px", - "pb-px", - "pl-px", - "p-0.5", - "px-0.5", - "py-0.5", - "ps-0.5", - "pe-0.5", - "pt-0.5", - "pr-0.5", - "pb-0.5", - "pl-0.5", - "p-1", - "px-1", - "py-1", - "ps-1", - "pe-1", - "pt-1", - "pr-1", - "pb-1", - "pl-1", - "p-1.5", - "px-1.5", - "py-1.5", - "ps-1.5", - "pe-1.5", - "pt-1.5", - "pr-1.5", - "pb-1.5", - "pl-1.5", - "p-2", - "px-2", - "py-2", - "ps-2", - "pe-2", - "pt-2", - "pr-2", - "pb-2", - "pl-2", - "p-2.5", - "px-2.5", - "py-2.5", - "ps-2.5", - "pe-2.5", - "pt-2.5", - "pr-2.5", - "pb-2.5", - "pl-2.5", - "p-3", - "px-3", - "py-3", - "ps-3", - "pe-3", - "pt-3", - "pr-3", - "pb-3", - "pl-3", - "p-3.5", - "px-3.5", - "py-3.5", - "ps-3.5", - "pe-3.5", - "pt-3.5", - "pr-3.5", - "pb-3.5", - "pl-3.5", - "p-4", - "px-4", - "py-4", - "ps-4", - "pe-4", - "pt-4", - "pr-4", - "pb-4", - "pl-4", - "p-5", - "px-5", - "py-5", - "ps-5", - "pe-5", - "pt-5", - "pr-5", - "pb-5", - "pl-5", - "p-6", - "px-6", - "py-6", - "ps-6", - "pe-6", - "pt-6", - "pr-6", - "pb-6", - "pl-6", - "p-7", - "px-7", - "py-7", - "ps-7", - "pe-7", - "pt-7", - "pr-7", - "pb-7", - "pl-7", - "p-8", - "px-8", - "py-8", - "ps-8", - "pe-8", - "pt-8", - "pr-8", - "pb-8", - "pl-8", - "p-9", - "px-9", - "py-9", - "ps-9", - "pe-9", - "pt-9", - "pr-9", - "pb-9", - "pl-9", - "p-10", - "px-10", - "py-10", - "ps-10", - "pe-10", - "pt-10", - "pr-10", - "pb-10", - "pl-10", - "p-11", - "px-11", - "py-11", - "ps-11", - "pe-11", - "pt-11", - "pr-11", - "pb-11", - "pl-11", - "p-12", - "px-12", - "py-12", - "ps-12", - "pe-12", - "pt-12", - "pr-12", - "pb-12", - "pl-12", - "p-14", - "px-14", - "py-14", - "ps-14", - "pe-14", - "pt-14", - "pr-14", - "pb-14", - "pl-14", - "p-16", - "px-16", - "py-16", - "ps-16", - "pe-16", - "pt-16", - "pr-16", - "pb-16", - "pl-16", - "p-20", - "px-20", - "py-20", - "ps-20", - "pe-20", - "pt-20", - "pr-20", - "pb-20", - "pl-20", - "p-24", - "px-24", - "py-24", - "ps-24", - "pe-24", - "pt-24", - "pr-24", - "pb-24", - "pl-24", - "p-28", - "px-28", - "py-28", - "ps-28", - "pe-28", - "pt-28", - "pr-28", - "pb-28", - "pl-28", - "p-32", - "px-32", - "py-32", - "ps-32", - "pe-32", - "pt-32", - "pr-32", - "pb-32", - "pl-32", - "p-36", - "px-36", - "py-36", - "ps-36", - "pe-36", - "pt-36", - "pr-36", - "pb-36", - "pl-36", - "p-40", - "px-40", - "py-40", - "ps-40", - "pe-40", - "pt-40", - "pr-40", - "pb-40", - "pl-40", - "p-44", - "px-44", - "py-44", - "ps-44", - "pe-44", - "pt-44", - "pr-44", - "pb-44", - "pl-44", - "p-48", - "px-48", - "py-48", - "ps-48", - "pe-48", - "pt-48", - "pr-48", - "pb-48", - "pl-48", - "p-52", - "px-52", - "py-52", - "ps-52", - "pe-52", - "pt-52", - "pr-52", - "pb-52", - "pl-52", - "p-56", - "px-56", - "py-56", - "ps-56", - "pe-56", - "pt-56", - "pr-56", - "pb-56", - "pl-56", - "p-60", - "px-60", - "py-60", - "ps-60", - "pe-60", - "pt-60", - "pr-60", - "pb-60", - "pl-60", - "p-64", - "px-64", - "py-64", - "ps-64", - "pe-64", - "pt-64", - "pr-64", - "pb-64", - "pl-64", - "p-72", - "px-72", - "py-72", - "ps-72", - "pe-72", - "pt-72", - "pr-72", - "pb-72", - "pl-72", - "p-80", - "px-80", - "py-80", - "ps-80", - "pe-80", - "pt-80", - "pr-80", - "pb-80", - "pl-80", - "p-96", - "px-96", - "py-96", - "ps-96", - "pe-96", - "pt-96", - "pr-96", - "pb-96", - "pl-96", - "m-0", - "mx-0", - "my-0", - "ms-0", - "me-0", - "mt-0", - "mr-0", - "mb-0", - "ml-0", - "m-px", - "mx-px", - "my-px", - "ms-px", - "me-px", - "mt-px", - "mr-px", - "mb-px", - "ml-px", - "m-0.5", - "mx-0.5", - "my-0.5", - "ms-0.5", - "me-0.5", - "mt-0.5", - "mr-0.5", - "mb-0.5", - "ml-0.5", - "m-1", - "mx-1", - "my-1", - "ms-1", - "me-1", - "mt-1", - "mr-1", - "mb-1", - "ml-1", - "m-1.5", - "mx-1.5", - "my-1.5", - "ms-1.5", - "me-1.5", - "mt-1.5", - "mr-1.5", - "mb-1.5", - "ml-1.5", - "m-2", - "mx-2", - "my-2", - "ms-2", - "me-2", - "mt-2", - "mr-2", - "mb-2", - "ml-2", - "m-2.5", - "mx-2.5", - "my-2.5", - "ms-2.5", - "me-2.5", - "mt-2.5", - "mr-2.5", - "mb-2.5", - "ml-2.5", - "m-3", - "mx-3", - "my-3", - "ms-3", - "me-3", - "mt-3", - "mr-3", - "mb-3", - "ml-3", - "m-3.5", - "mx-3.5", - "my-3.5", - "ms-3.5", - "me-3.5", - "mt-3.5", - "mr-3.5", - "mb-3.5", - "ml-3.5", - "m-4", - "mx-4", - "my-4", - "ms-4", - "me-4", - "mt-4", - "mr-4", - "mb-4", - "ml-4", - "m-5", - "mx-5", - "my-5", - "ms-5", - "me-5", - "mt-5", - "mr-5", - "mb-5", - "ml-5", - "m-6", - "mx-6", - "my-6", - "ms-6", - "me-6", - "mt-6", - "mr-6", - "mb-6", - "ml-6", - "m-7", - "mx-7", - "my-7", - "ms-7", - "me-7", - "mt-7", - "mr-7", - "mb-7", - "ml-7", - "m-8", - "mx-8", - "my-8", - "ms-8", - "me-8", - "mt-8", - "mr-8", - "mb-8", - "ml-8", - "m-9", - "mx-9", - "my-9", - "ms-9", - "me-9", - "mt-9", - "mr-9", - "mb-9", - "ml-9", - "m-10", - "mx-10", - "my-10", - "ms-10", - "me-10", - "mt-10", - "mr-10", - "mb-10", - "ml-10", - "m-11", - "mx-11", - "my-11", - "ms-11", - "me-11", - "mt-11", - "mr-11", - "mb-11", - "ml-11", - "m-12", - "mx-12", - "my-12", - "ms-12", - "me-12", - "mt-12", - "mr-12", - "mb-12", - "ml-12", - "m-14", - "mx-14", - "my-14", - "ms-14", - "me-14", - "mt-14", - "mr-14", - "mb-14", - "ml-14", - "m-16", - "mx-16", - "my-16", - "ms-16", - "me-16", - "mt-16", - "mr-16", - "mb-16", - "ml-16", - "m-20", - "mx-20", - "my-20", - "ms-20", - "me-20", - "mt-20", - "mr-20", - "mb-20", - "ml-20", - "m-24", - "mx-24", - "my-24", - "ms-24", - "me-24", - "mt-24", - "mr-24", - "mb-24", - "ml-24", - "m-28", - "mx-28", - "my-28", - "ms-28", - "me-28", - "mt-28", - "mr-28", - "mb-28", - "ml-28", - "m-32", - "mx-32", - "my-32", - "ms-32", - "me-32", - "mt-32", - "mr-32", - "mb-32", - "ml-32", - "m-36", - "mx-36", - "my-36", - "ms-36", - "me-36", - "mt-36", - "mr-36", - "mb-36", - "ml-36", - "m-40", - "mx-40", - "my-40", - "ms-40", - "me-40", - "mt-40", - "mr-40", - "mb-40", - "ml-40", - "m-44", - "mx-44", - "my-44", - "ms-44", - "me-44", - "mt-44", - "mr-44", - "mb-44", - "ml-44", - "m-48", - "mx-48", - "my-48", - "ms-48", - "me-48", - "mt-48", - "mr-48", - "mb-48", - "ml-48", - "m-52", - "mx-52", - "my-52", - "ms-52", - "me-52", - "mt-52", - "mr-52", - "mb-52", - "ml-52", - "m-56", - "mx-56", - "my-56", - "ms-56", - "me-56", - "mt-56", - "mr-56", - "mb-56", - "ml-56", - "m-60", - "mx-60", - "my-60", - "ms-60", - "me-60", - "mt-60", - "mr-60", - "mb-60", - "ml-60", - "m-64", - "mx-64", - "my-64", - "ms-64", - "me-64", - "mt-64", - "mr-64", - "mb-64", - "ml-64", - "m-72", - "mx-72", - "my-72", - "ms-72", - "me-72", - "mt-72", - "mr-72", - "mb-72", - "ml-72", - "m-80", - "mx-80", - "my-80", - "ms-80", - "me-80", - "mt-80", - "mr-80", - "mb-80", - "ml-80", - "m-96", - "mx-96", - "my-96", - "ms-96", - "me-96", - "mt-96", - "mr-96", - "mb-96", - "ml-96", - "m-auto", - "mx-auto", - "my-auto", - "ms-auto", - "me-auto", - "mt-auto", - "mr-auto", - "mb-auto", - "ml-auto", - "space-x-0", - "space-y-0", - "space-x-0.5", - "space-y-0.5", - "space-x-1", - "space-y-1", - "space-x-1.5", - "space-y-1.5", - "space-x-2", - "space-y-2", - "space-x-2.5", - "space-y-2.5", - "space-x-3", - "space-y-3", - "space-x-3.5", - "space-y-3.5", - "space-x-4", - "space-y-4", - "space-x-5", - "space-y-5", - "space-x-6", - "space-y-6", - "space-x-7", - "space-y-7", - "space-x-8", - "space-y-8", - "space-x-9", - "space-y-9", - "space-x-10", - "space-y-10", - "space-x-11", - "space-y-11", - "space-x-12", - "space-y-12", - "space-x-14", - "space-y-14", - "space-x-16", - "space-y-16", - "space-x-20", - "space-y-20", - "space-x-24", - "space-y-24", - "space-x-28", - "space-y-28", - "space-x-32", - "space-y-32", - "space-x-36", - "space-y-36", - "space-x-40", - "space-y-40", - "space-x-44", - "space-y-44", - "space-x-48", - "space-y-48", - "space-x-52", - "space-y-52", - "space-x-56", - "space-y-56", - "space-x-60", - "space-y-60", - "space-x-64", - "space-y-64", - "space-x-72", - "space-y-72", - "space-x-80", - "space-y-80", - "space-x-96", - "space-y-96", - "space-x-px", - "space-y-px", - "space-y-reverse", - "space-x-reverse", - "w-0", - "w-px", - "w-0.5", - "w-1", - "w-1.5", - "w-2", - "w-2.5", - "w-3", - "w-3.5", - "w-4", - "w-5", - "w-6", - "w-7", - "w-8", - "w-9", - "w-10", - "w-11", - "w-12", - "w-14", - "w-16", - "w-20", - "w-24", - "w-28", - "w-32", - "w-36", - "w-40", - "w-44", - "w-48", - "w-52", - "w-56", - "w-60", - "w-64", - "w-72", - "w-80", - "w-96", - "w-auto", - "w-1/2", - "w-1/3", - "w-2/3", - "w-1/4", - "w-2/4", - "w-3/4", - "w-1/5", - "w-2/5", - "w-3/5", - "w-4/5", - "w-1/6", - "w-2/6", - "w-3/6", - "w-4/6", - "w-5/6", - "w-1/12", - "w-2/12", - "w-3/12", - "w-4/12", - "w-5/12", - "w-6/12", - "w-7/12", - "w-8/12", - "w-9/12", - "w-10/12", - "w-11/12", - "w-full", - "w-screen", - "w-svw", - "w-lvw", - "w-dvw", - "w-min", - "w-max", - "w-fit", - "min-w-0", - "min-w-1", - "min-w-2", - "min-w-3", - "min-w-4", - "min-w-5", - "min-w-6", - "min-w-7", - "min-w-8", - "min-w-9", - "min-w-10", - "min-w-11", - "min-w-12", - "min-w-14", - "min-w-16", - "min-w-20", - "min-w-24", - "min-w-28", - "min-w-32", - "min-w-36", - "min-w-40", - "min-w-44", - "min-w-48", - "min-w-52", - "min-w-56", - "min-w-60", - "min-w-64", - "min-w-72", - "min-w-80", - "min-w-96", - "min-w-px", - "min-w-0.5", - "min-w-1.5", - "min-w-2.5", - "min-w-3.5", - "min-w-full", - "min-w-min", - "min-w-max", - "min-w-fit", - "max-w-0", - "max-w-px", - "max-w-0.5", - "max-w-1", - "max-w-1.5", - "max-w-2", - "max-w-2.5", - "max-w-3", - "max-w-3.5", - "max-w-4", - "max-w-5", - "max-w-6", - "max-w-7", - "max-w-8", - "max-w-9", - "max-w-10", - "max-w-11", - "max-w-12", - "max-w-14", - "max-w-16", - "max-w-20", - "max-w-24", - "max-w-28", - "max-w-32", - "max-w-36", - "max-w-40", - "max-w-44", - "max-w-48", - "max-w-52", - "max-w-56", - "max-w-60", - "max-w-64", - "max-w-72", - "max-w-80", - "max-w-96", - "max-w-none", - "max-w-xs", - "max-w-sm", - "max-w-md", - "max-w-lg", - "max-w-xl", - "max-w-2xl", - "max-w-3xl", - "max-w-4xl", - "max-w-5xl", - "max-w-6xl", - "max-w-7xl", - "max-w-full", - "max-w-min", - "max-w-max", - "max-w-fit", - "max-w-prose", - "max-w-screen-sm", - "max-w-screen-md", - "max-w-screen-lg", - "max-w-screen-xl", - "max-w-screen-2xl", - "h-0", - "h-px", - "h-0.5", - "h-1", - "h-1.5", - "h-2", - "h-2.5", - "h-3", - "h-3.5", - "h-4", - "h-5", - "h-6", - "h-7", - "h-8", - "h-9", - "h-10", - "h-11", - "h-12", - "h-14", - "h-16", - "h-20", - "h-24", - "h-28", - "h-32", - "h-36", - "h-40", - "h-44", - "h-48", - "h-52", - "h-56", - "h-60", - "h-64", - "h-72", - "h-80", - "h-96", - "h-auto", - "h-1/2", - "h-1/3", - "h-2/3", - "h-1/4", - "h-2/4", - "h-3/4", - "h-1/5", - "h-2/5", - "h-3/5", - "h-4/5", - "h-1/6", - "h-2/6", - "h-3/6", - "h-4/6", - "h-5/6", - "h-full", - "h-screen", - "h-svh", - "h-lvh", - "h-dvh", - "h-min", - "h-max", - "h-fit", - "min-h-0", - "min-h-1", - "min-h-2", - "min-h-3", - "min-h-4", - "min-h-5", - "min-h-6", - "min-h-7", - "min-h-8", - "min-h-9", - "min-h-10", - "min-h-11", - "min-h-12", - "min-h-14", - "min-h-16", - "min-h-20", - "min-h-24", - "min-h-28", - "min-h-32", - "min-h-36", - "min-h-40", - "min-h-44", - "min-h-48", - "min-h-52", - "min-h-56", - "min-h-60", - "min-h-64", - "min-h-72", - "min-h-80", - "min-h-96", - "min-h-px", - "min-h-0.5", - "min-h-1.5", - "min-h-2.5", - "min-h-3.5", - "min-h-full", - "min-h-screen", - "min-h-svh", - "min-h-lvh", - "min-h-dvh", - "min-h-min", - "min-h-max", - "min-h-fit", - "max-h-0", - "max-h-px", - "max-h-0.5", - "max-h-1", - "max-h-1.5", - "max-h-2", - "max-h-2.5", - "max-h-3", - "max-h-3.5", - "max-h-4", - "max-h-5", - "max-h-6", - "max-h-7", - "max-h-8", - "max-h-9", - "max-h-10", - "max-h-11", - "max-h-12", - "max-h-14", - "max-h-16", - "max-h-20", - "max-h-24", - "max-h-28", - "max-h-32", - "max-h-36", - "max-h-40", - "max-h-44", - "max-h-48", - "max-h-52", - "max-h-56", - "max-h-60", - "max-h-64", - "max-h-72", - "max-h-80", - "max-h-96", - "max-h-none", - "max-h-full", - "max-h-screen", - "max-h-svh", - "max-h-lvh", - "max-h-dvh", - "max-h-min", - "max-h-max", - "max-h-fit", - "size-0", - "size-px", - "size-0.5", - "size-1", - "size-1.5", - "size-2", - "size-2.5", - "size-3", - "size-3.5", - "size-4", - "size-5", - "size-6", - "size-7", - "size-8", - "size-9", - "size-10", - "size-11", - "size-12", - "size-14", - "size-16", - "size-20", - "size-24", - "size-28", - "size-32", - "size-36", - "size-40", - "size-44", - "size-48", - "size-52", - "size-56", - "size-60", - "size-64", - "size-72", - "size-80", - "size-96", - "size-auto", - "size-1/2", - "size-1/3", - "size-2/3", - "size-1/4", - "size-2/4", - "size-3/4", - "size-1/5", - "size-2/5", - "size-3/5", - "size-4/5", - "size-1/6", - "size-2/6", - "size-3/6", - "size-4/6", - "size-5/6", - "size-1/12", - "size-2/12", - "size-3/12", - "size-4/12", - "size-5/12", - "size-6/12", - "size-7/12", - "size-8/12", - "size-9/12", - "size-10/12", - "size-11/12", - "size-full", - "size-min", - "size-max", - "size-fit", - "font-sans", - "font-serif", - "font-mono", - "text-xs", - "text-sm", - "text-base", - "text-lg", - "text-xl", - "text-2xl", - "text-3xl", - "text-4xl", - "text-5xl", - "text-6xl", - "text-7xl", - "text-8xl", - "text-9xl", - "antialiased", - "subpixel-antialiased", - "italic", - "not-italic", - "font-thin", - "font-extralight", - "font-light", - "font-normal", - "font-medium", - "font-semibold", - "font-bold", - "font-extrabold", - "font-black", - "normal-nums", - "ordinal", - "slashed-zero", - "lining-nums", - "oldstyle-nums", - "proportional-nums", - "tabular-nums", - "diagonal-fractions", - "stacked-fractions", - "tracking-tighter", - "tracking-tight", - "tracking-normal", - "tracking-wide", - "tracking-wider", - "tracking-widest", - "line-clamp-1", - "line-clamp-2", - "line-clamp-3", - "line-clamp-4", - "line-clamp-5", - "line-clamp-6", - "line-clamp-none", - "leading-3", - "leading-4", - "leading-5", - "leading-6", - "leading-7", - "leading-8", - "leading-9", - "leading-10", - "leading-none", - "leading-tight", - "leading-snug", - "leading-normal", - "leading-relaxed", - "leading-loose", - "list-image-none", - "list-inside", - "list-outside", - "list-none", - "list-disc", - "list-decimal", - "text-left", - "text-center", - "text-right", - "text-justify", - "text-start", - "text-end", - "text-inherit", - "text-current", - "text-transparent", - "text-black", - "text-white", - "text-slate-50", - "text-slate-100", - "text-slate-200", - "text-slate-300", - "text-slate-400", - "text-slate-500", - "text-slate-600", - "text-slate-700", - "text-slate-800", - "text-slate-900", - "text-slate-950", - "text-gray-50", - "text-gray-100", - "text-gray-200", - "text-gray-300", - "text-gray-400", - "text-gray-500", - "text-gray-600", - "text-gray-700", - "text-gray-800", - "text-gray-900", - "text-gray-950", - "text-zinc-50", - "text-zinc-100", - "text-zinc-200", - "text-zinc-300", - "text-zinc-400", - "text-zinc-500", - "text-zinc-600", - "text-zinc-700", - "text-zinc-800", - "text-zinc-900", - "text-zinc-950", - "text-neutral-50", - "text-neutral-100", - "text-neutral-200", - "text-neutral-300", - "text-neutral-400", - "text-neutral-500", - "text-neutral-600", - "text-neutral-700", - "text-neutral-800", - "text-neutral-900", - "text-neutral-950", - "text-stone-50", - "text-stone-100", - "text-stone-200", - "text-stone-300", - "text-stone-400", - "text-stone-500", - "text-stone-600", - "text-stone-700", - "text-stone-800", - "text-stone-900", - "text-stone-950", - "text-red-50", - "text-red-100", - "text-red-200", - "text-red-300", - "text-red-400", - "text-red-500", - "text-red-600", - "text-red-700", - "text-red-800", - "text-red-900", - "text-red-950", - "text-orange-50", - "text-orange-100", - "text-orange-200", - "text-orange-300", - "text-orange-400", - "text-orange-500", - "text-orange-600", - "text-orange-700", - "text-orange-800", - "text-orange-900", - "text-orange-950", - "text-amber-50", - "text-amber-100", - "text-amber-200", - "text-amber-300", - "text-amber-400", - "text-amber-500", - "text-amber-600", - "text-amber-700", - "text-amber-800", - "text-amber-900", - "text-amber-950", - "text-yellow-50", - "text-yellow-100", - "text-yellow-200", - "text-yellow-300", - "text-yellow-400", - "text-yellow-500", - "text-yellow-600", - "text-yellow-700", - "text-yellow-800", - "text-yellow-900", - "text-yellow-950", - "text-lime-50", - "text-lime-100", - "text-lime-200", - "text-lime-300", - "text-lime-400", - "text-lime-500", - "text-lime-600", - "text-lime-700", - "text-lime-800", - "text-lime-900", - "text-lime-950", - "text-green-50", - "text-green-100", - "text-green-200", - "text-green-300", - "text-green-400", - "text-green-500", - "text-green-600", - "text-green-700", - "text-green-800", - "text-green-900", - "text-green-950", - "text-emerald-50", - "text-emerald-100", - "text-emerald-200", - "text-emerald-300", - "text-emerald-400", - "text-emerald-500", - "text-emerald-600", - "text-emerald-700", - "text-emerald-800", - "text-emerald-900", - "text-emerald-950", - "text-teal-50", - "text-teal-100", - "text-teal-200", - "text-teal-300", - "text-teal-400", - "text-teal-500", - "text-teal-600", - "text-teal-700", - "text-teal-800", - "text-teal-900", - "text-teal-950", - "text-cyan-50", - "text-cyan-100", - "text-cyan-200", - "text-cyan-300", - "text-cyan-400", - "text-cyan-500", - "text-cyan-600", - "text-cyan-700", - "text-cyan-800", - "text-cyan-900", - "text-cyan-950", - "text-sky-50", - "text-sky-100", - "text-sky-200", - "text-sky-300", - "text-sky-400", - "text-sky-500", - "text-sky-600", - "text-sky-700", - "text-sky-800", - "text-sky-900", - "text-sky-950", - "text-blue-50", - "text-blue-100", - "text-blue-200", - "text-blue-300", - "text-blue-400", - "text-blue-500", - "text-blue-600", - "text-blue-700", - "text-blue-800", - "text-blue-900", - "text-blue-950", - "text-indigo-50", - "text-indigo-100", - "text-indigo-200", - "text-indigo-300", - "text-indigo-400", - "text-indigo-500", - "text-indigo-600", - "text-indigo-700", - "text-indigo-800", - "text-indigo-900", - "text-indigo-950", - "text-violet-50", - "text-violet-100", - "text-violet-200", - "text-violet-300", - "text-violet-400", - "text-violet-500", - "text-violet-600", - "text-violet-700", - "text-violet-800", - "text-violet-900", - "text-violet-950", - "text-purple-50", - "text-purple-100", - "text-purple-200", - "text-purple-300", - "text-purple-400", - "text-purple-500", - "text-purple-600", - "text-purple-700", - "text-purple-800", - "text-purple-900", - "text-purple-950", - "text-fuchsia-50", - "text-fuchsia-100", - "text-fuchsia-200", - "text-fuchsia-300", - "text-fuchsia-400", - "text-fuchsia-500", - "text-fuchsia-600", - "text-fuchsia-700", - "text-fuchsia-800", - "text-fuchsia-900", - "text-fuchsia-950", - "text-pink-50", - "text-pink-100", - "text-pink-200", - "text-pink-300", - "text-pink-400", - "text-pink-500", - "text-pink-600", - "text-pink-700", - "text-pink-800", - "text-pink-900", - "text-pink-950", - "text-rose-50", - "text-rose-100", - "text-rose-200", - "text-rose-300", - "text-rose-400", - "text-rose-500", - "text-rose-600", - "text-rose-700", - "text-rose-800", - "text-rose-900", - "text-rose-950", - "underline", - "overline", - "line-through", - "no-underline", - "decoration-inherit", - "decoration-current", - "decoration-transparent", - "decoration-black", - "decoration-white", - "decoration-slate-50", - "decoration-slate-100", - "decoration-slate-200", - "decoration-slate-300", - "decoration-slate-400", - "decoration-slate-500", - "decoration-slate-600", - "decoration-slate-700", - "decoration-slate-800", - "decoration-slate-900", - "decoration-slate-950", - "decoration-gray-50", - "decoration-gray-100", - "decoration-gray-200", - "decoration-gray-300", - "decoration-gray-400", - "decoration-gray-500", - "decoration-gray-600", - "decoration-gray-700", - "decoration-gray-800", - "decoration-gray-900", - "decoration-gray-950", - "decoration-zinc-50", - "decoration-zinc-100", - "decoration-zinc-200", - "decoration-zinc-300", - "decoration-zinc-400", - "decoration-zinc-500", - "decoration-zinc-600", - "decoration-zinc-700", - "decoration-zinc-800", - "decoration-zinc-900", - "decoration-zinc-950", - "decoration-neutral-50", - "decoration-neutral-100", - "decoration-neutral-200", - "decoration-neutral-300", - "decoration-neutral-400", - "decoration-neutral-500", - "decoration-neutral-600", - "decoration-neutral-700", - "decoration-neutral-800", - "decoration-neutral-900", - "decoration-neutral-950", - "decoration-stone-50", - "decoration-stone-100", - "decoration-stone-200", - "decoration-stone-300", - "decoration-stone-400", - "decoration-stone-500", - "decoration-stone-600", - "decoration-stone-700", - "decoration-stone-800", - "decoration-stone-900", - "decoration-stone-950", - "decoration-red-50", - "decoration-red-100", - "decoration-red-200", - "decoration-red-300", - "decoration-red-400", - "decoration-red-500", - "decoration-red-600", - "decoration-red-700", - "decoration-red-800", - "decoration-red-900", - "decoration-red-950", - "decoration-orange-50", - "decoration-orange-100", - "decoration-orange-200", - "decoration-orange-300", - "decoration-orange-400", - "decoration-orange-500", - "decoration-orange-600", - "decoration-orange-700", - "decoration-orange-800", - "decoration-orange-900", - "decoration-orange-950", - "decoration-amber-50", - "decoration-amber-100", - "decoration-amber-200", - "decoration-amber-300", - "decoration-amber-400", - "decoration-amber-500", - "decoration-amber-600", - "decoration-amber-700", - "decoration-amber-800", - "decoration-amber-900", - "decoration-amber-950", - "decoration-yellow-50", - "decoration-yellow-100", - "decoration-yellow-200", - "decoration-yellow-300", - "decoration-yellow-400", - "decoration-yellow-500", - "decoration-yellow-600", - "decoration-yellow-700", - "decoration-yellow-800", - "decoration-yellow-900", - "decoration-yellow-950", - "decoration-lime-50", - "decoration-lime-100", - "decoration-lime-200", - "decoration-lime-300", - "decoration-lime-400", - "decoration-lime-500", - "decoration-lime-600", - "decoration-lime-700", - "decoration-lime-800", - "decoration-lime-900", - "decoration-lime-950", - "decoration-green-50", - "decoration-green-100", - "decoration-green-200", - "decoration-green-300", - "decoration-green-400", - "decoration-green-500", - "decoration-green-600", - "decoration-green-700", - "decoration-green-800", - "decoration-green-900", - "decoration-green-950", - "decoration-emerald-50", - "decoration-emerald-100", - "decoration-emerald-200", - "decoration-emerald-300", - "decoration-emerald-400", - "decoration-emerald-500", - "decoration-emerald-600", - "decoration-emerald-700", - "decoration-emerald-800", - "decoration-emerald-900", - "decoration-emerald-950", - "decoration-teal-50", - "decoration-teal-100", - "decoration-teal-200", - "decoration-teal-300", - "decoration-teal-400", - "decoration-teal-500", - "decoration-teal-600", - "decoration-teal-700", - "decoration-teal-800", - "decoration-teal-900", - "decoration-teal-950", - "decoration-cyan-50", - "decoration-cyan-100", - "decoration-cyan-200", - "decoration-cyan-300", - "decoration-cyan-400", - "decoration-cyan-500", - "decoration-cyan-600", - "decoration-cyan-700", - "decoration-cyan-800", - "decoration-cyan-900", - "decoration-cyan-950", - "decoration-sky-50", - "decoration-sky-100", - "decoration-sky-200", - "decoration-sky-300", - "decoration-sky-400", - "decoration-sky-500", - "decoration-sky-600", - "decoration-sky-700", - "decoration-sky-800", - "decoration-sky-900", - "decoration-sky-950", - "decoration-blue-50", - "decoration-blue-100", - "decoration-blue-200", - "decoration-blue-300", - "decoration-blue-400", - "decoration-blue-500", - "decoration-blue-600", - "decoration-blue-700", - "decoration-blue-800", - "decoration-blue-900", - "decoration-blue-950", - "decoration-indigo-50", - "decoration-indigo-100", - "decoration-indigo-200", - "decoration-indigo-300", - "decoration-indigo-400", - "decoration-indigo-500", - "decoration-indigo-600", - "decoration-indigo-700", - "decoration-indigo-800", - "decoration-indigo-900", - "decoration-indigo-950", - "decoration-violet-50", - "decoration-violet-100", - "decoration-violet-200", - "decoration-violet-300", - "decoration-violet-400", - "decoration-violet-500", - "decoration-violet-600", - "decoration-violet-700", - "decoration-violet-800", - "decoration-violet-900", - "decoration-violet-950", - "decoration-purple-50", - "decoration-purple-100", - "decoration-purple-200", - "decoration-purple-300", - "decoration-purple-400", - "decoration-purple-500", - "decoration-purple-600", - "decoration-purple-700", - "decoration-purple-800", - "decoration-purple-900", - "decoration-purple-950", - "decoration-fuchsia-50", - "decoration-fuchsia-100", - "decoration-fuchsia-200", - "decoration-fuchsia-300", - "decoration-fuchsia-400", - "decoration-fuchsia-500", - "decoration-fuchsia-600", - "decoration-fuchsia-700", - "decoration-fuchsia-800", - "decoration-fuchsia-900", - "decoration-fuchsia-950", - "decoration-pink-50", - "decoration-pink-100", - "decoration-pink-200", - "decoration-pink-300", - "decoration-pink-400", - "decoration-pink-500", - "decoration-pink-600", - "decoration-pink-700", - "decoration-pink-800", - "decoration-pink-900", - "decoration-pink-950", - "decoration-rose-50", - "decoration-rose-100", - "decoration-rose-200", - "decoration-rose-300", - "decoration-rose-400", - "decoration-rose-500", - "decoration-rose-600", - "decoration-rose-700", - "decoration-rose-800", - "decoration-rose-900", - "decoration-rose-950", - "decoration-solid", - "decoration-double", - "decoration-dotted", - "decoration-dashed", - "decoration-wavy", - "decoration-auto", - "decoration-from-font", - "decoration-0", - "decoration-1", - "decoration-2", - "decoration-4", - "decoration-8", - "underline-offset-auto", - "underline-offset-0", - "underline-offset-1", - "underline-offset-2", - "underline-offset-4", - "underline-offset-8", - "uppercase", - "lowercase", - "capitalize", - "normal-case", - "truncate", - "text-ellipsis", - "text-clip", - "text-wrap", - "text-nowrap", - "text-balance", - "text-pretty", - "indent-0", - "indent-px", - "indent-0.5", - "indent-1", - "indent-1.5", - "indent-2", - "indent-2.5", - "indent-3", - "indent-3.5", - "indent-4", - "indent-5", - "indent-6", - "indent-7", - "indent-8", - "indent-9", - "indent-10", - "indent-11", - "indent-12", - "indent-14", - "indent-16", - "indent-20", - "indent-24", - "indent-28", - "indent-32", - "indent-36", - "indent-40", - "indent-44", - "indent-48", - "indent-52", - "indent-56", - "indent-60", - "indent-64", - "indent-72", - "indent-80", - "indent-96", - "align-baseline", - "align-top", - "align-middle", - "align-bottom", - "align-text-top", - "align-text-bottom", - "align-sub", - "align-super", - "whitespace-normal", - "whitespace-nowrap", - "whitespace-pre", - "whitespace-pre-line", - "whitespace-pre-wrap", - "whitespace-break-spaces", - "break-normal", - "break-words", - "break-all", - "break-keep", - "hyphens-none", - "hyphens-manual", - "hyphens-auto", - "content-none", - "bg-fixed", - "bg-local", - "bg-scroll", - "bg-clip-border", - "bg-clip-padding", - "bg-clip-content", - "bg-clip-text", - "bg-inherit", - "bg-current", - "bg-transparent", - "bg-black", - "bg-white", - "bg-slate-50", - "bg-slate-100", - "bg-slate-200", - "bg-slate-300", - "bg-slate-400", - "bg-slate-500", - "bg-slate-600", - "bg-slate-700", - "bg-slate-800", - "bg-slate-900", - "bg-slate-950", - "bg-gray-50", - "bg-gray-100", - "bg-gray-200", - "bg-gray-300", - "bg-gray-400", - "bg-gray-500", - "bg-gray-600", - "bg-gray-700", - "bg-gray-800", - "bg-gray-900", - "bg-gray-950", - "bg-zinc-50", - "bg-zinc-100", - "bg-zinc-200", - "bg-zinc-300", - "bg-zinc-400", - "bg-zinc-500", - "bg-zinc-600", - "bg-zinc-700", - "bg-zinc-800", - "bg-zinc-900", - "bg-zinc-950", - "bg-neutral-50", - "bg-neutral-100", - "bg-neutral-200", - "bg-neutral-300", - "bg-neutral-400", - "bg-neutral-500", - "bg-neutral-600", - "bg-neutral-700", - "bg-neutral-800", - "bg-neutral-900", - "bg-neutral-950", - "bg-stone-50", - "bg-stone-100", - "bg-stone-200", - "bg-stone-300", - "bg-stone-400", - "bg-stone-500", - "bg-stone-600", - "bg-stone-700", - "bg-stone-800", - "bg-stone-900", - "bg-stone-950", - "bg-red-50", - "bg-red-100", - "bg-red-200", - "bg-red-300", - "bg-red-400", - "bg-red-500", - "bg-red-600", - "bg-red-700", - "bg-red-800", - "bg-red-900", - "bg-red-950", - "bg-orange-50", - "bg-orange-100", - "bg-orange-200", - "bg-orange-300", - "bg-orange-400", - "bg-orange-500", - "bg-orange-600", - "bg-orange-700", - "bg-orange-800", - "bg-orange-900", - "bg-orange-950", - "bg-amber-50", - "bg-amber-100", - "bg-amber-200", - "bg-amber-300", - "bg-amber-400", - "bg-amber-500", - "bg-amber-600", - "bg-amber-700", - "bg-amber-800", - "bg-amber-900", - "bg-amber-950", - "bg-yellow-50", - "bg-yellow-100", - "bg-yellow-200", - "bg-yellow-300", - "bg-yellow-400", - "bg-yellow-500", - "bg-yellow-600", - "bg-yellow-700", - "bg-yellow-800", - "bg-yellow-900", - "bg-yellow-950", - "bg-lime-50", - "bg-lime-100", - "bg-lime-200", - "bg-lime-300", - "bg-lime-400", - "bg-lime-500", - "bg-lime-600", - "bg-lime-700", - "bg-lime-800", - "bg-lime-900", - "bg-lime-950", - "bg-green-50", - "bg-green-100", - "bg-green-200", - "bg-green-300", - "bg-green-400", - "bg-green-500", - "bg-green-600", - "bg-green-700", - "bg-green-800", - "bg-green-900", - "bg-green-950", - "bg-emerald-50", - "bg-emerald-100", - "bg-emerald-200", - "bg-emerald-300", - "bg-emerald-400", - "bg-emerald-500", - "bg-emerald-600", - "bg-emerald-700", - "bg-emerald-800", - "bg-emerald-900", - "bg-emerald-950", - "bg-teal-50", - "bg-teal-100", - "bg-teal-200", - "bg-teal-300", - "bg-teal-400", - "bg-teal-500", - "bg-teal-600", - "bg-teal-700", - "bg-teal-800", - "bg-teal-900", - "bg-teal-950", - "bg-cyan-50", - "bg-cyan-100", - "bg-cyan-200", - "bg-cyan-300", - "bg-cyan-400", - "bg-cyan-500", - "bg-cyan-600", - "bg-cyan-700", - "bg-cyan-800", - "bg-cyan-900", - "bg-cyan-950", - "bg-sky-50", - "bg-sky-100", - "bg-sky-200", - "bg-sky-300", - "bg-sky-400", - "bg-sky-500", - "bg-sky-600", - "bg-sky-700", - "bg-sky-800", - "bg-sky-900", - "bg-sky-950", - "bg-blue-50", - "bg-blue-100", - "bg-blue-200", - "bg-blue-300", - "bg-blue-400", - "bg-blue-500", - "bg-blue-600", - "bg-blue-700", - "bg-blue-800", - "bg-blue-900", - "bg-blue-950", - "bg-indigo-50", - "bg-indigo-100", - "bg-indigo-200", - "bg-indigo-300", - "bg-indigo-400", - "bg-indigo-500", - "bg-indigo-600", - "bg-indigo-700", - "bg-indigo-800", - "bg-indigo-900", - "bg-indigo-950", - "bg-violet-50", - "bg-violet-100", - "bg-violet-200", - "bg-violet-300", - "bg-violet-400", - "bg-violet-500", - "bg-violet-600", - "bg-violet-700", - "bg-violet-800", - "bg-violet-900", - "bg-violet-950", - "bg-purple-50", - "bg-purple-100", - "bg-purple-200", - "bg-purple-300", - "bg-purple-400", - "bg-purple-500", - "bg-purple-600", - "bg-purple-700", - "bg-purple-800", - "bg-purple-900", - "bg-purple-950", - "bg-fuchsia-50", - "bg-fuchsia-100", - "bg-fuchsia-200", - "bg-fuchsia-300", - "bg-fuchsia-400", - "bg-fuchsia-500", - "bg-fuchsia-600", - "bg-fuchsia-700", - "bg-fuchsia-800", - "bg-fuchsia-900", - "bg-fuchsia-950", - "bg-pink-50", - "bg-pink-100", - "bg-pink-200", - "bg-pink-300", - "bg-pink-400", - "bg-pink-500", - "bg-pink-600", - "bg-pink-700", - "bg-pink-800", - "bg-pink-900", - "bg-pink-950", - "bg-rose-50", - "bg-rose-100", - "bg-rose-200", - "bg-rose-300", - "bg-rose-400", - "bg-rose-500", - "bg-rose-600", - "bg-rose-700", - "bg-rose-800", - "bg-rose-900", - "bg-rose-950", - "bg-origin-border", - "bg-origin-padding", - "bg-origin-content", - "bg-bottom", - "bg-center", - "bg-left", - "bg-left-bottom", - "bg-left-top", - "bg-right", - "bg-right-bottom", - "bg-right-top", - "bg-top", - "bg-repeat", - "bg-no-repeat", - "bg-repeat-x", - "bg-repeat-y", - "bg-repeat-round", - "bg-repeat-space", - "bg-auto", - "bg-cover", - "bg-contain", - "bg-none", - "bg-gradient-to-t", - "bg-gradient-to-tr", - "bg-gradient-to-r", - "bg-gradient-to-br", - "bg-gradient-to-b", - "bg-gradient-to-bl", - "bg-gradient-to-l", - "bg-gradient-to-tl", - "from-inherit", - "from-current", - "from-transparent", - "from-black", - "from-white", - "from-slate-50", - "from-slate-100", - "from-slate-200", - "from-slate-300", - "from-slate-400", - "from-slate-500", - "from-slate-600", - "from-slate-700", - "from-slate-800", - "from-slate-900", - "from-slate-950", - "from-gray-50", - "from-gray-100", - "from-gray-200", - "from-gray-300", - "from-gray-400", - "from-gray-500", - "from-gray-600", - "from-gray-700", - "from-gray-800", - "from-gray-900", - "from-gray-950", - "from-zinc-50", - "from-zinc-100", - "from-zinc-200", - "from-zinc-300", - "from-zinc-400", - "from-zinc-500", - "from-zinc-600", - "from-zinc-700", - "from-zinc-800", - "from-zinc-900", - "from-zinc-950", - "from-neutral-50", - "from-neutral-100", - "from-neutral-200", - "from-neutral-300", - "from-neutral-400", - "from-neutral-500", - "from-neutral-600", - "from-neutral-700", - "from-neutral-800", - "from-neutral-900", - "from-neutral-950", - "from-stone-50", - "from-stone-100", - "from-stone-200", - "from-stone-300", - "from-stone-400", - "from-stone-500", - "from-stone-600", - "from-stone-700", - "from-stone-800", - "from-stone-900", - "from-stone-950", - "from-red-50", - "from-red-100", - "from-red-200", - "from-red-300", - "from-red-400", - "from-red-500", - "from-red-600", - "from-red-700", - "from-red-800", - "from-red-900", - "from-red-950", - "from-orange-50", - "from-orange-100", - "from-orange-200", - "from-orange-300", - "from-orange-400", - "from-orange-500", - "from-orange-600", - "from-orange-700", - "from-orange-800", - "from-orange-900", - "from-orange-950", - "from-amber-50", - "from-amber-100", - "from-amber-200", - "from-amber-300", - "from-amber-400", - "from-amber-500", - "from-amber-600", - "from-amber-700", - "from-amber-800", - "from-amber-900", - "from-amber-950", - "from-yellow-50", - "from-yellow-100", - "from-yellow-200", - "from-yellow-300", - "from-yellow-400", - "from-yellow-500", - "from-yellow-600", - "from-yellow-700", - "from-yellow-800", - "from-yellow-900", - "from-yellow-950", - "from-lime-50", - "from-lime-100", - "from-lime-200", - "from-lime-300", - "from-lime-400", - "from-lime-500", - "from-lime-600", - "from-lime-700", - "from-lime-800", - "from-lime-900", - "from-lime-950", - "from-green-50", - "from-green-100", - "from-green-200", - "from-green-300", - "from-green-400", - "from-green-500", - "from-green-600", - "from-green-700", - "from-green-800", - "from-green-900", - "from-green-950", - "from-emerald-50", - "from-emerald-100", - "from-emerald-200", - "from-emerald-300", - "from-emerald-400", - "from-emerald-500", - "from-emerald-600", - "from-emerald-700", - "from-emerald-800", - "from-emerald-900", - "from-emerald-950", - "from-teal-50", - "from-teal-100", - "from-teal-200", - "from-teal-300", - "from-teal-400", - "from-teal-500", - "from-teal-600", - "from-teal-700", - "from-teal-800", - "from-teal-900", - "from-teal-950", - "from-cyan-50", - "from-cyan-100", - "from-cyan-200", - "from-cyan-300", - "from-cyan-400", - "from-cyan-500", - "from-cyan-600", - "from-cyan-700", - "from-cyan-800", - "from-cyan-900", - "from-cyan-950", - "from-sky-50", - "from-sky-100", - "from-sky-200", - "from-sky-300", - "from-sky-400", - "from-sky-500", - "from-sky-600", - "from-sky-700", - "from-sky-800", - "from-sky-900", - "from-sky-950", - "from-blue-50", - "from-blue-100", - "from-blue-200", - "from-blue-300", - "from-blue-400", - "from-blue-500", - "from-blue-600", - "from-blue-700", - "from-blue-800", - "from-blue-900", - "from-blue-950", - "from-indigo-50", - "from-indigo-100", - "from-indigo-200", - "from-indigo-300", - "from-indigo-400", - "from-indigo-500", - "from-indigo-600", - "from-indigo-700", - "from-indigo-800", - "from-indigo-900", - "from-indigo-950", - "from-violet-50", - "from-violet-100", - "from-violet-200", - "from-violet-300", - "from-violet-400", - "from-violet-500", - "from-violet-600", - "from-violet-700", - "from-violet-800", - "from-violet-900", - "from-violet-950", - "from-purple-50", - "from-purple-100", - "from-purple-200", - "from-purple-300", - "from-purple-400", - "from-purple-500", - "from-purple-600", - "from-purple-700", - "from-purple-800", - "from-purple-900", - "from-purple-950", - "from-fuchsia-50", - "from-fuchsia-100", - "from-fuchsia-200", - "from-fuchsia-300", - "from-fuchsia-400", - "from-fuchsia-500", - "from-fuchsia-600", - "from-fuchsia-700", - "from-fuchsia-800", - "from-fuchsia-900", - "from-fuchsia-950", - "from-pink-50", - "from-pink-100", - "from-pink-200", - "from-pink-300", - "from-pink-400", - "from-pink-500", - "from-pink-600", - "from-pink-700", - "from-pink-800", - "from-pink-900", - "from-pink-950", - "from-rose-50", - "from-rose-100", - "from-rose-200", - "from-rose-300", - "from-rose-400", - "from-rose-500", - "from-rose-600", - "from-rose-700", - "from-rose-800", - "from-rose-900", - "from-rose-950", - "from-0%", - "from-5%", - "from-10%", - "from-15%", - "from-20%", - "from-25%", - "from-30%", - "from-35%", - "from-40%", - "from-45%", - "from-50%", - "from-55%", - "from-60%", - "from-65%", - "from-70%", - "from-75%", - "from-80%", - "from-85%", - "from-90%", - "from-95%", - "from-100%", - "via-inherit", - "via-current", - "via-transparent", - "via-black", - "via-white", - "via-slate-50", - "via-slate-100", - "via-slate-200", - "via-slate-300", - "via-slate-400", - "via-slate-500", - "via-slate-600", - "via-slate-700", - "via-slate-800", - "via-slate-900", - "via-slate-950", - "via-gray-50", - "via-gray-100", - "via-gray-200", - "via-gray-300", - "via-gray-400", - "via-gray-500", - "via-gray-600", - "via-gray-700", - "via-gray-800", - "via-gray-900", - "via-gray-950", - "via-zinc-50", - "via-zinc-100", - "via-zinc-200", - "via-zinc-300", - "via-zinc-400", - "via-zinc-500", - "via-zinc-600", - "via-zinc-700", - "via-zinc-800", - "via-zinc-900", - "via-zinc-950", - "via-neutral-50", - "via-neutral-100", - "via-neutral-200", - "via-neutral-300", - "via-neutral-400", - "via-neutral-500", - "via-neutral-600", - "via-neutral-700", - "via-neutral-800", - "via-neutral-900", - "via-neutral-950", - "via-stone-50", - "via-stone-100", - "via-stone-200", - "via-stone-300", - "via-stone-400", - "via-stone-500", - "via-stone-600", - "via-stone-700", - "via-stone-800", - "via-stone-900", - "via-stone-950", - "via-red-50", - "via-red-100", - "via-red-200", - "via-red-300", - "via-red-400", - "via-red-500", - "via-red-600", - "via-red-700", - "via-red-800", - "via-red-900", - "via-red-950", - "via-orange-50", - "via-orange-100", - "via-orange-200", - "via-orange-300", - "via-orange-400", - "via-orange-500", - "via-orange-600", - "via-orange-700", - "via-orange-800", - "via-orange-900", - "via-orange-950", - "via-amber-50", - "via-amber-100", - "via-amber-200", - "via-amber-300", - "via-amber-400", - "via-amber-500", - "via-amber-600", - "via-amber-700", - "via-amber-800", - "via-amber-900", - "via-amber-950", - "via-yellow-50", - "via-yellow-100", - "via-yellow-200", - "via-yellow-300", - "via-yellow-400", - "via-yellow-500", - "via-yellow-600", - "via-yellow-700", - "via-yellow-800", - "via-yellow-900", - "via-yellow-950", - "via-lime-50", - "via-lime-100", - "via-lime-200", - "via-lime-300", - "via-lime-400", - "via-lime-500", - "via-lime-600", - "via-lime-700", - "via-lime-800", - "via-lime-900", - "via-lime-950", - "via-green-50", - "via-green-100", - "via-green-200", - "via-green-300", - "via-green-400", - "via-green-500", - "via-green-600", - "via-green-700", - "via-green-800", - "via-green-900", - "via-green-950", - "via-emerald-50", - "via-emerald-100", - "via-emerald-200", - "via-emerald-300", - "via-emerald-400", - "via-emerald-500", - "via-emerald-600", - "via-emerald-700", - "via-emerald-800", - "via-emerald-900", - "via-emerald-950", - "via-teal-50", - "via-teal-100", - "via-teal-200", - "via-teal-300", - "via-teal-400", - "via-teal-500", - "via-teal-600", - "via-teal-700", - "via-teal-800", - "via-teal-900", - "via-teal-950", - "via-cyan-50", - "via-cyan-100", - "via-cyan-200", - "via-cyan-300", - "via-cyan-400", - "via-cyan-500", - "via-cyan-600", - "via-cyan-700", - "via-cyan-800", - "via-cyan-900", - "via-cyan-950", - "via-sky-50", - "via-sky-100", - "via-sky-200", - "via-sky-300", - "via-sky-400", - "via-sky-500", - "via-sky-600", - "via-sky-700", - "via-sky-800", - "via-sky-900", - "via-sky-950", - "via-blue-50", - "via-blue-100", - "via-blue-200", - "via-blue-300", - "via-blue-400", - "via-blue-500", - "via-blue-600", - "via-blue-700", - "via-blue-800", - "via-blue-900", - "via-blue-950", - "via-indigo-50", - "via-indigo-100", - "via-indigo-200", - "via-indigo-300", - "via-indigo-400", - "via-indigo-500", - "via-indigo-600", - "via-indigo-700", - "via-indigo-800", - "via-indigo-900", - "via-indigo-950", - "via-violet-50", - "via-violet-100", - "via-violet-200", - "via-violet-300", - "via-violet-400", - "via-violet-500", - "via-violet-600", - "via-violet-700", - "via-violet-800", - "via-violet-900", - "via-violet-950", - "via-purple-50", - "via-purple-100", - "via-purple-200", - "via-purple-300", - "via-purple-400", - "via-purple-500", - "via-purple-600", - "via-purple-700", - "via-purple-800", - "via-purple-900", - "via-purple-950", - "via-fuchsia-50", - "via-fuchsia-100", - "via-fuchsia-200", - "via-fuchsia-300", - "via-fuchsia-400", - "via-fuchsia-500", - "via-fuchsia-600", - "via-fuchsia-700", - "via-fuchsia-800", - "via-fuchsia-900", - "via-fuchsia-950", - "via-pink-50", - "via-pink-100", - "via-pink-200", - "via-pink-300", - "via-pink-400", - "via-pink-500", - "via-pink-600", - "via-pink-700", - "via-pink-800", - "via-pink-900", - "via-pink-950", - "via-rose-50", - "via-rose-100", - "via-rose-200", - "via-rose-300", - "via-rose-400", - "via-rose-500", - "via-rose-600", - "via-rose-700", - "via-rose-800", - "via-rose-900", - "via-rose-950", - "via-0%", - "via-5%", - "via-10%", - "via-15%", - "via-20%", - "via-25%", - "via-30%", - "via-35%", - "via-40%", - "via-45%", - "via-50%", - "via-55%", - "via-60%", - "via-65%", - "via-70%", - "via-75%", - "via-80%", - "via-85%", - "via-90%", - "via-95%", - "via-100%", - "to-inherit", - "to-current", - "to-transparent", - "to-black", - "to-white", - "to-slate-50", - "to-slate-100", - "to-slate-200", - "to-slate-300", - "to-slate-400", - "to-slate-500", - "to-slate-600", - "to-slate-700", - "to-slate-800", - "to-slate-900", - "to-slate-950", - "to-gray-50", - "to-gray-100", - "to-gray-200", - "to-gray-300", - "to-gray-400", - "to-gray-500", - "to-gray-600", - "to-gray-700", - "to-gray-800", - "to-gray-900", - "to-gray-950", - "to-zinc-50", - "to-zinc-100", - "to-zinc-200", - "to-zinc-300", - "to-zinc-400", - "to-zinc-500", - "to-zinc-600", - "to-zinc-700", - "to-zinc-800", - "to-zinc-900", - "to-zinc-950", - "to-neutral-50", - "to-neutral-100", - "to-neutral-200", - "to-neutral-300", - "to-neutral-400", - "to-neutral-500", - "to-neutral-600", - "to-neutral-700", - "to-neutral-800", - "to-neutral-900", - "to-neutral-950", - "to-stone-50", - "to-stone-100", - "to-stone-200", - "to-stone-300", - "to-stone-400", - "to-stone-500", - "to-stone-600", - "to-stone-700", - "to-stone-800", - "to-stone-900", - "to-stone-950", - "to-red-50", - "to-red-100", - "to-red-200", - "to-red-300", - "to-red-400", - "to-red-500", - "to-red-600", - "to-red-700", - "to-red-800", - "to-red-900", - "to-red-950", - "to-orange-50", - "to-orange-100", - "to-orange-200", - "to-orange-300", - "to-orange-400", - "to-orange-500", - "to-orange-600", - "to-orange-700", - "to-orange-800", - "to-orange-900", - "to-orange-950", - "to-amber-50", - "to-amber-100", - "to-amber-200", - "to-amber-300", - "to-amber-400", - "to-amber-500", - "to-amber-600", - "to-amber-700", - "to-amber-800", - "to-amber-900", - "to-amber-950", - "to-yellow-50", - "to-yellow-100", - "to-yellow-200", - "to-yellow-300", - "to-yellow-400", - "to-yellow-500", - "to-yellow-600", - "to-yellow-700", - "to-yellow-800", - "to-yellow-900", - "to-yellow-950", - "to-lime-50", - "to-lime-100", - "to-lime-200", - "to-lime-300", - "to-lime-400", - "to-lime-500", - "to-lime-600", - "to-lime-700", - "to-lime-800", - "to-lime-900", - "to-lime-950", - "to-green-50", - "to-green-100", - "to-green-200", - "to-green-300", - "to-green-400", - "to-green-500", - "to-green-600", - "to-green-700", - "to-green-800", - "to-green-900", - "to-green-950", - "to-emerald-50", - "to-emerald-100", - "to-emerald-200", - "to-emerald-300", - "to-emerald-400", - "to-emerald-500", - "to-emerald-600", - "to-emerald-700", - "to-emerald-800", - "to-emerald-900", - "to-emerald-950", - "to-teal-50", - "to-teal-100", - "to-teal-200", - "to-teal-300", - "to-teal-400", - "to-teal-500", - "to-teal-600", - "to-teal-700", - "to-teal-800", - "to-teal-900", - "to-teal-950", - "to-cyan-50", - "to-cyan-100", - "to-cyan-200", - "to-cyan-300", - "to-cyan-400", - "to-cyan-500", - "to-cyan-600", - "to-cyan-700", - "to-cyan-800", - "to-cyan-900", - "to-cyan-950", - "to-sky-50", - "to-sky-100", - "to-sky-200", - "to-sky-300", - "to-sky-400", - "to-sky-500", - "to-sky-600", - "to-sky-700", - "to-sky-800", - "to-sky-900", - "to-sky-950", - "to-blue-50", - "to-blue-100", - "to-blue-200", - "to-blue-300", - "to-blue-400", - "to-blue-500", - "to-blue-600", - "to-blue-700", - "to-blue-800", - "to-blue-900", - "to-blue-950", - "to-indigo-50", - "to-indigo-100", - "to-indigo-200", - "to-indigo-300", - "to-indigo-400", - "to-indigo-500", - "to-indigo-600", - "to-indigo-700", - "to-indigo-800", - "to-indigo-900", - "to-indigo-950", - "to-violet-50", - "to-violet-100", - "to-violet-200", - "to-violet-300", - "to-violet-400", - "to-violet-500", - "to-violet-600", - "to-violet-700", - "to-violet-800", - "to-violet-900", - "to-violet-950", - "to-purple-50", - "to-purple-100", - "to-purple-200", - "to-purple-300", - "to-purple-400", - "to-purple-500", - "to-purple-600", - "to-purple-700", - "to-purple-800", - "to-purple-900", - "to-purple-950", - "to-fuchsia-50", - "to-fuchsia-100", - "to-fuchsia-200", - "to-fuchsia-300", - "to-fuchsia-400", - "to-fuchsia-500", - "to-fuchsia-600", - "to-fuchsia-700", - "to-fuchsia-800", - "to-fuchsia-900", - "to-fuchsia-950", - "to-pink-50", - "to-pink-100", - "to-pink-200", - "to-pink-300", - "to-pink-400", - "to-pink-500", - "to-pink-600", - "to-pink-700", - "to-pink-800", - "to-pink-900", - "to-pink-950", - "to-rose-50", - "to-rose-100", - "to-rose-200", - "to-rose-300", - "to-rose-400", - "to-rose-500", - "to-rose-600", - "to-rose-700", - "to-rose-800", - "to-rose-900", - "to-rose-950", - "to-0%", - "to-5%", - "to-10%", - "to-15%", - "to-20%", - "to-25%", - "to-30%", - "to-35%", - "to-40%", - "to-45%", - "to-50%", - "to-55%", - "to-60%", - "to-65%", - "to-70%", - "to-75%", - "to-80%", - "to-85%", - "to-90%", - "to-95%", - "to-100%", - "rounded-none", - "rounded-sm", - "rounded", - "rounded-md", - "rounded-lg", - "rounded-xl", - "rounded-2xl", - "rounded-3xl", - "rounded-full", - "rounded-s-none", - "rounded-s-sm", - "rounded-s", - "rounded-s-md", - "rounded-s-lg", - "rounded-s-xl", - "rounded-s-2xl", - "rounded-s-3xl", - "rounded-s-full", - "rounded-e-none", - "rounded-e-sm", - "rounded-e", - "rounded-e-md", - "rounded-e-lg", - "rounded-e-xl", - "rounded-e-2xl", - "rounded-e-3xl", - "rounded-e-full", - "rounded-t-none", - "rounded-t-sm", - "rounded-t", - "rounded-t-md", - "rounded-t-lg", - "rounded-t-xl", - "rounded-t-2xl", - "rounded-t-3xl", - "rounded-t-full", - "rounded-r-none", - "rounded-r-sm", - "rounded-r", - "rounded-r-md", - "rounded-r-lg", - "rounded-r-xl", - "rounded-r-2xl", - "rounded-r-3xl", - "rounded-r-full", - "rounded-b-none", - "rounded-b-sm", - "rounded-b", - "rounded-b-md", - "rounded-b-lg", - "rounded-b-xl", - "rounded-b-2xl", - "rounded-b-3xl", - "rounded-b-full", - "rounded-l-none", - "rounded-l-sm", - "rounded-l", - "rounded-l-md", - "rounded-l-lg", - "rounded-l-xl", - "rounded-l-2xl", - "rounded-l-3xl", - "rounded-l-full", - "rounded-ss-none", - "rounded-ss-sm", - "rounded-ss", - "rounded-ss-md", - "rounded-ss-lg", - "rounded-ss-xl", - "rounded-ss-2xl", - "rounded-ss-3xl", - "rounded-ss-full", - "rounded-se-none", - "rounded-se-sm", - "rounded-se", - "rounded-se-md", - "rounded-se-lg", - "rounded-se-xl", - "rounded-se-2xl", - "rounded-se-3xl", - "rounded-se-full", - "rounded-ee-none", - "rounded-ee-sm", - "rounded-ee", - "rounded-ee-md", - "rounded-ee-lg", - "rounded-ee-xl", - "rounded-ee-2xl", - "rounded-ee-3xl", - "rounded-ee-full", - "rounded-es-none", - "rounded-es-sm", - "rounded-es", - "rounded-es-md", - "rounded-es-lg", - "rounded-es-xl", - "rounded-es-2xl", - "rounded-es-3xl", - "rounded-es-full", - "rounded-tl-none", - "rounded-tl-sm", - "rounded-tl", - "rounded-tl-md", - "rounded-tl-lg", - "rounded-tl-xl", - "rounded-tl-2xl", - "rounded-tl-3xl", - "rounded-tl-full", - "rounded-tr-none", - "rounded-tr-sm", - "rounded-tr", - "rounded-tr-md", - "rounded-tr-lg", - "rounded-tr-xl", - "rounded-tr-2xl", - "rounded-tr-3xl", - "rounded-tr-full", - "rounded-br-none", - "rounded-br-sm", - "rounded-br", - "rounded-br-md", - "rounded-br-lg", - "rounded-br-xl", - "rounded-br-2xl", - "rounded-br-3xl", - "rounded-br-full", - "rounded-bl-none", - "rounded-bl-sm", - "rounded-bl", - "rounded-bl-md", - "rounded-bl-lg", - "rounded-bl-xl", - "rounded-bl-2xl", - "rounded-bl-3xl", - "rounded-bl-full", - "rounded-s-*", - "rounded-e-*", - "rounded-ss-*", - "rounded-se-*", - "rounded-es-*", - "rounded-ee-*", - "border-0", - "border-2", - "border-4", - "border-8", - "border", - "border-x-0", - "border-x-2", - "border-x-4", - "border-x-8", - "border-x", - "border-y-0", - "border-y-2", - "border-y-4", - "border-y-8", - "border-y", - "border-s-0", - "border-s-2", - "border-s-4", - "border-s-8", - "border-s", - "border-e-0", - "border-e-2", - "border-e-4", - "border-e-8", - "border-e", - "border-t-0", - "border-t-2", - "border-t-4", - "border-t-8", - "border-t", - "border-r-0", - "border-r-2", - "border-r-4", - "border-r-8", - "border-r", - "border-b-0", - "border-b-2", - "border-b-4", - "border-b-8", - "border-b", - "border-l-0", - "border-l-2", - "border-l-4", - "border-l-8", - "border-l", - "border-inherit", - "border-current", - "border-transparent", - "border-black", - "border-white", - "border-slate-50", - "border-slate-100", - "border-slate-200", - "border-slate-300", - "border-slate-400", - "border-slate-500", - "border-slate-600", - "border-slate-700", - "border-slate-800", - "border-slate-900", - "border-slate-950", - "border-gray-50", - "border-gray-100", - "border-gray-200", - "border-gray-300", - "border-gray-400", - "border-gray-500", - "border-gray-600", - "border-gray-700", - "border-gray-800", - "border-gray-900", - "border-gray-950", - "border-zinc-50", - "border-zinc-100", - "border-zinc-200", - "border-zinc-300", - "border-zinc-400", - "border-zinc-500", - "border-zinc-600", - "border-zinc-700", - "border-zinc-800", - "border-zinc-900", - "border-zinc-950", - "border-neutral-50", - "border-neutral-100", - "border-neutral-200", - "border-neutral-300", - "border-neutral-400", - "border-neutral-500", - "border-neutral-600", - "border-neutral-700", - "border-neutral-800", - "border-neutral-900", - "border-neutral-950", - "border-stone-50", - "border-stone-100", - "border-stone-200", - "border-stone-300", - "border-stone-400", - "border-stone-500", - "border-stone-600", - "border-stone-700", - "border-stone-800", - "border-stone-900", - "border-stone-950", - "border-red-50", - "border-red-100", - "border-red-200", - "border-red-300", - "border-red-400", - "border-red-500", - "border-red-600", - "border-red-700", - "border-red-800", - "border-red-900", - "border-red-950", - "border-orange-50", - "border-orange-100", - "border-orange-200", - "border-orange-300", - "border-orange-400", - "border-orange-500", - "border-orange-600", - "border-orange-700", - "border-orange-800", - "border-orange-900", - "border-orange-950", - "border-amber-50", - "border-amber-100", - "border-amber-200", - "border-amber-300", - "border-amber-400", - "border-amber-500", - "border-amber-600", - "border-amber-700", - "border-amber-800", - "border-amber-900", - "border-amber-950", - "border-yellow-50", - "border-yellow-100", - "border-yellow-200", - "border-yellow-300", - "border-yellow-400", - "border-yellow-500", - "border-yellow-600", - "border-yellow-700", - "border-yellow-800", - "border-yellow-900", - "border-yellow-950", - "border-lime-50", - "border-lime-100", - "border-lime-200", - "border-lime-300", - "border-lime-400", - "border-lime-500", - "border-lime-600", - "border-lime-700", - "border-lime-800", - "border-lime-900", - "border-lime-950", - "border-green-50", - "border-green-100", - "border-green-200", - "border-green-300", - "border-green-400", - "border-green-500", - "border-green-600", - "border-green-700", - "border-green-800", - "border-green-900", - "border-green-950", - "border-emerald-50", - "border-emerald-100", - "border-emerald-200", - "border-emerald-300", - "border-emerald-400", - "border-emerald-500", - "border-emerald-600", - "border-emerald-700", - "border-emerald-800", - "border-emerald-900", - "border-emerald-950", - "border-teal-50", - "border-teal-100", - "border-teal-200", - "border-teal-300", - "border-teal-400", - "border-teal-500", - "border-teal-600", - "border-teal-700", - "border-teal-800", - "border-teal-900", - "border-teal-950", - "border-cyan-50", - "border-cyan-100", - "border-cyan-200", - "border-cyan-300", - "border-cyan-400", - "border-cyan-500", - "border-cyan-600", - "border-cyan-700", - "border-cyan-800", - "border-cyan-900", - "border-cyan-950", - "border-sky-50", - "border-sky-100", - "border-sky-200", - "border-sky-300", - "border-sky-400", - "border-sky-500", - "border-sky-600", - "border-sky-700", - "border-sky-800", - "border-sky-900", - "border-sky-950", - "border-blue-50", - "border-blue-100", - "border-blue-200", - "border-blue-300", - "border-blue-400", - "border-blue-500", - "border-blue-600", - "border-blue-700", - "border-blue-800", - "border-blue-900", - "border-blue-950", - "border-indigo-50", - "border-indigo-100", - "border-indigo-200", - "border-indigo-300", - "border-indigo-400", - "border-indigo-500", - "border-indigo-600", - "border-indigo-700", - "border-indigo-800", - "border-indigo-900", - "border-indigo-950", - "border-violet-50", - "border-violet-100", - "border-violet-200", - "border-violet-300", - "border-violet-400", - "border-violet-500", - "border-violet-600", - "border-violet-700", - "border-violet-800", - "border-violet-900", - "border-violet-950", - "border-purple-50", - "border-purple-100", - "border-purple-200", - "border-purple-300", - "border-purple-400", - "border-purple-500", - "border-purple-600", - "border-purple-700", - "border-purple-800", - "border-purple-900", - "border-purple-950", - "border-fuchsia-50", - "border-fuchsia-100", - "border-fuchsia-200", - "border-fuchsia-300", - "border-fuchsia-400", - "border-fuchsia-500", - "border-fuchsia-600", - "border-fuchsia-700", - "border-fuchsia-800", - "border-fuchsia-900", - "border-fuchsia-950", - "border-pink-50", - "border-pink-100", - "border-pink-200", - "border-pink-300", - "border-pink-400", - "border-pink-500", - "border-pink-600", - "border-pink-700", - "border-pink-800", - "border-pink-900", - "border-pink-950", - "border-rose-50", - "border-rose-100", - "border-rose-200", - "border-rose-300", - "border-rose-400", - "border-rose-500", - "border-rose-600", - "border-rose-700", - "border-rose-800", - "border-rose-900", - "border-rose-950", - "border-x-inherit", - "border-x-current", - "border-x-transparent", - "border-x-black", - "border-x-white", - "border-x-slate-50", - "border-x-slate-100", - "border-x-slate-200", - "border-x-slate-300", - "border-x-slate-400", - "border-x-slate-500", - "border-x-slate-600", - "border-x-slate-700", - "border-x-slate-800", - "border-x-slate-900", - "border-x-slate-950", - "border-x-gray-50", - "border-x-gray-100", - "border-x-gray-200", - "border-x-gray-300", - "border-x-gray-400", - "border-x-gray-500", - "border-x-gray-600", - "border-x-gray-700", - "border-x-gray-800", - "border-x-gray-900", - "border-x-gray-950", - "border-x-zinc-50", - "border-x-zinc-100", - "border-x-zinc-200", - "border-x-zinc-300", - "border-x-zinc-400", - "border-x-zinc-500", - "border-x-zinc-600", - "border-x-zinc-700", - "border-x-zinc-800", - "border-x-zinc-900", - "border-x-zinc-950", - "border-x-neutral-50", - "border-x-neutral-100", - "border-x-neutral-200", - "border-x-neutral-300", - "border-x-neutral-400", - "border-x-neutral-500", - "border-x-neutral-600", - "border-x-neutral-700", - "border-x-neutral-800", - "border-x-neutral-900", - "border-x-neutral-950", - "border-x-stone-50", - "border-x-stone-100", - "border-x-stone-200", - "border-x-stone-300", - "border-x-stone-400", - "border-x-stone-500", - "border-x-stone-600", - "border-x-stone-700", - "border-x-stone-800", - "border-x-stone-900", - "border-x-stone-950", - "border-x-red-50", - "border-x-red-100", - "border-x-red-200", - "border-x-red-300", - "border-x-red-400", - "border-x-red-500", - "border-x-red-600", - "border-x-red-700", - "border-x-red-800", - "border-x-red-900", - "border-x-red-950", - "border-x-orange-50", - "border-x-orange-100", - "border-x-orange-200", - "border-x-orange-300", - "border-x-orange-400", - "border-x-orange-500", - "border-x-orange-600", - "border-x-orange-700", - "border-x-orange-800", - "border-x-orange-900", - "border-x-orange-950", - "border-x-amber-50", - "border-x-amber-100", - "border-x-amber-200", - "border-x-amber-300", - "border-x-amber-400", - "border-x-amber-500", - "border-x-amber-600", - "border-x-amber-700", - "border-x-amber-800", - "border-x-amber-900", - "border-x-amber-950", - "border-x-yellow-50", - "border-x-yellow-100", - "border-x-yellow-200", - "border-x-yellow-300", - "border-x-yellow-400", - "border-x-yellow-500", - "border-x-yellow-600", - "border-x-yellow-700", - "border-x-yellow-800", - "border-x-yellow-900", - "border-x-yellow-950", - "border-x-lime-50", - "border-x-lime-100", - "border-x-lime-200", - "border-x-lime-300", - "border-x-lime-400", - "border-x-lime-500", - "border-x-lime-600", - "border-x-lime-700", - "border-x-lime-800", - "border-x-lime-900", - "border-x-lime-950", - "border-x-green-50", - "border-x-green-100", - "border-x-green-200", - "border-x-green-300", - "border-x-green-400", - "border-x-green-500", - "border-x-green-600", - "border-x-green-700", - "border-x-green-800", - "border-x-green-900", - "border-x-green-950", - "border-x-emerald-50", - "border-x-emerald-100", - "border-x-emerald-200", - "border-x-emerald-300", - "border-x-emerald-400", - "border-x-emerald-500", - "border-x-emerald-600", - "border-x-emerald-700", - "border-x-emerald-800", - "border-x-emerald-900", - "border-x-emerald-950", - "border-x-teal-50", - "border-x-teal-100", - "border-x-teal-200", - "border-x-teal-300", - "border-x-teal-400", - "border-x-teal-500", - "border-x-teal-600", - "border-x-teal-700", - "border-x-teal-800", - "border-x-teal-900", - "border-x-teal-950", - "border-x-cyan-50", - "border-x-cyan-100", - "border-x-cyan-200", - "border-x-cyan-300", - "border-x-cyan-400", - "border-x-cyan-500", - "border-x-cyan-600", - "border-x-cyan-700", - "border-x-cyan-800", - "border-x-cyan-900", - "border-x-cyan-950", - "border-x-sky-50", - "border-x-sky-100", - "border-x-sky-200", - "border-x-sky-300", - "border-x-sky-400", - "border-x-sky-500", - "border-x-sky-600", - "border-x-sky-700", - "border-x-sky-800", - "border-x-sky-900", - "border-x-sky-950", - "border-x-blue-50", - "border-x-blue-100", - "border-x-blue-200", - "border-x-blue-300", - "border-x-blue-400", - "border-x-blue-500", - "border-x-blue-600", - "border-x-blue-700", - "border-x-blue-800", - "border-x-blue-900", - "border-x-blue-950", - "border-x-indigo-50", - "border-x-indigo-100", - "border-x-indigo-200", - "border-x-indigo-300", - "border-x-indigo-400", - "border-x-indigo-500", - "border-x-indigo-600", - "border-x-indigo-700", - "border-x-indigo-800", - "border-x-indigo-900", - "border-x-indigo-950", - "border-x-violet-50", - "border-x-violet-100", - "border-x-violet-200", - "border-x-violet-300", - "border-x-violet-400", - "border-x-violet-500", - "border-x-violet-600", - "border-x-violet-700", - "border-x-violet-800", - "border-x-violet-900", - "border-x-violet-950", - "border-x-purple-50", - "border-x-purple-100", - "border-x-purple-200", - "border-x-purple-300", - "border-x-purple-400", - "border-x-purple-500", - "border-x-purple-600", - "border-x-purple-700", - "border-x-purple-800", - "border-x-purple-900", - "border-x-purple-950", - "border-x-fuchsia-50", - "border-x-fuchsia-100", - "border-x-fuchsia-200", - "border-x-fuchsia-300", - "border-x-fuchsia-400", - "border-x-fuchsia-500", - "border-x-fuchsia-600", - "border-x-fuchsia-700", - "border-x-fuchsia-800", - "border-x-fuchsia-900", - "border-x-fuchsia-950", - "border-x-pink-50", - "border-x-pink-100", - "border-x-pink-200", - "border-x-pink-300", - "border-x-pink-400", - "border-x-pink-500", - "border-x-pink-600", - "border-x-pink-700", - "border-x-pink-800", - "border-x-pink-900", - "border-x-pink-950", - "border-x-rose-50", - "border-x-rose-100", - "border-x-rose-200", - "border-x-rose-300", - "border-x-rose-400", - "border-x-rose-500", - "border-x-rose-600", - "border-x-rose-700", - "border-x-rose-800", - "border-x-rose-900", - "border-x-rose-950", - "border-y-inherit", - "border-y-current", - "border-y-transparent", - "border-y-black", - "border-y-white", - "border-y-slate-50", - "border-y-slate-100", - "border-y-slate-200", - "border-y-slate-300", - "border-y-slate-400", - "border-y-slate-500", - "border-y-slate-600", - "border-y-slate-700", - "border-y-slate-800", - "border-y-slate-900", - "border-y-slate-950", - "border-y-gray-50", - "border-y-gray-100", - "border-y-gray-200", - "border-y-gray-300", - "border-y-gray-400", - "border-y-gray-500", - "border-y-gray-600", - "border-y-gray-700", - "border-y-gray-800", - "border-y-gray-900", - "border-y-gray-950", - "border-y-zinc-50", - "border-y-zinc-100", - "border-y-zinc-200", - "border-y-zinc-300", - "border-y-zinc-400", - "border-y-zinc-500", - "border-y-zinc-600", - "border-y-zinc-700", - "border-y-zinc-800", - "border-y-zinc-900", - "border-y-zinc-950", - "border-y-neutral-50", - "border-y-neutral-100", - "border-y-neutral-200", - "border-y-neutral-300", - "border-y-neutral-400", - "border-y-neutral-500", - "border-y-neutral-600", - "border-y-neutral-700", - "border-y-neutral-800", - "border-y-neutral-900", - "border-y-neutral-950", - "border-y-stone-50", - "border-y-stone-100", - "border-y-stone-200", - "border-y-stone-300", - "border-y-stone-400", - "border-y-stone-500", - "border-y-stone-600", - "border-y-stone-700", - "border-y-stone-800", - "border-y-stone-900", - "border-y-stone-950", - "border-y-red-50", - "border-y-red-100", - "border-y-red-200", - "border-y-red-300", - "border-y-red-400", - "border-y-red-500", - "border-y-red-600", - "border-y-red-700", - "border-y-red-800", - "border-y-red-900", - "border-y-red-950", - "border-y-orange-50", - "border-y-orange-100", - "border-y-orange-200", - "border-y-orange-300", - "border-y-orange-400", - "border-y-orange-500", - "border-y-orange-600", - "border-y-orange-700", - "border-y-orange-800", - "border-y-orange-900", - "border-y-orange-950", - "border-y-amber-50", - "border-y-amber-100", - "border-y-amber-200", - "border-y-amber-300", - "border-y-amber-400", - "border-y-amber-500", - "border-y-amber-600", - "border-y-amber-700", - "border-y-amber-800", - "border-y-amber-900", - "border-y-amber-950", - "border-y-yellow-50", - "border-y-yellow-100", - "border-y-yellow-200", - "border-y-yellow-300", - "border-y-yellow-400", - "border-y-yellow-500", - "border-y-yellow-600", - "border-y-yellow-700", - "border-y-yellow-800", - "border-y-yellow-900", - "border-y-yellow-950", - "border-y-lime-50", - "border-y-lime-100", - "border-y-lime-200", - "border-y-lime-300", - "border-y-lime-400", - "border-y-lime-500", - "border-y-lime-600", - "border-y-lime-700", - "border-y-lime-800", - "border-y-lime-900", - "border-y-lime-950", - "border-y-green-50", - "border-y-green-100", - "border-y-green-200", - "border-y-green-300", - "border-y-green-400", - "border-y-green-500", - "border-y-green-600", - "border-y-green-700", - "border-y-green-800", - "border-y-green-900", - "border-y-green-950", - "border-y-emerald-50", - "border-y-emerald-100", - "border-y-emerald-200", - "border-y-emerald-300", - "border-y-emerald-400", - "border-y-emerald-500", - "border-y-emerald-600", - "border-y-emerald-700", - "border-y-emerald-800", - "border-y-emerald-900", - "border-y-emerald-950", - "border-y-teal-50", - "border-y-teal-100", - "border-y-teal-200", - "border-y-teal-300", - "border-y-teal-400", - "border-y-teal-500", - "border-y-teal-600", - "border-y-teal-700", - "border-y-teal-800", - "border-y-teal-900", - "border-y-teal-950", - "border-y-cyan-50", - "border-y-cyan-100", - "border-y-cyan-200", - "border-y-cyan-300", - "border-y-cyan-400", - "border-y-cyan-500", - "border-y-cyan-600", - "border-y-cyan-700", - "border-y-cyan-800", - "border-y-cyan-900", - "border-y-cyan-950", - "border-y-sky-50", - "border-y-sky-100", - "border-y-sky-200", - "border-y-sky-300", - "border-y-sky-400", - "border-y-sky-500", - "border-y-sky-600", - "border-y-sky-700", - "border-y-sky-800", - "border-y-sky-900", - "border-y-sky-950", - "border-y-blue-50", - "border-y-blue-100", - "border-y-blue-200", - "border-y-blue-300", - "border-y-blue-400", - "border-y-blue-500", - "border-y-blue-600", - "border-y-blue-700", - "border-y-blue-800", - "border-y-blue-900", - "border-y-blue-950", - "border-y-indigo-50", - "border-y-indigo-100", - "border-y-indigo-200", - "border-y-indigo-300", - "border-y-indigo-400", - "border-y-indigo-500", - "border-y-indigo-600", - "border-y-indigo-700", - "border-y-indigo-800", - "border-y-indigo-900", - "border-y-indigo-950", - "border-y-violet-50", - "border-y-violet-100", - "border-y-violet-200", - "border-y-violet-300", - "border-y-violet-400", - "border-y-violet-500", - "border-y-violet-600", - "border-y-violet-700", - "border-y-violet-800", - "border-y-violet-900", - "border-y-violet-950", - "border-y-purple-50", - "border-y-purple-100", - "border-y-purple-200", - "border-y-purple-300", - "border-y-purple-400", - "border-y-purple-500", - "border-y-purple-600", - "border-y-purple-700", - "border-y-purple-800", - "border-y-purple-900", - "border-y-purple-950", - "border-y-fuchsia-50", - "border-y-fuchsia-100", - "border-y-fuchsia-200", - "border-y-fuchsia-300", - "border-y-fuchsia-400", - "border-y-fuchsia-500", - "border-y-fuchsia-600", - "border-y-fuchsia-700", - "border-y-fuchsia-800", - "border-y-fuchsia-900", - "border-y-fuchsia-950", - "border-y-pink-50", - "border-y-pink-100", - "border-y-pink-200", - "border-y-pink-300", - "border-y-pink-400", - "border-y-pink-500", - "border-y-pink-600", - "border-y-pink-700", - "border-y-pink-800", - "border-y-pink-900", - "border-y-pink-950", - "border-y-rose-50", - "border-y-rose-100", - "border-y-rose-200", - "border-y-rose-300", - "border-y-rose-400", - "border-y-rose-500", - "border-y-rose-600", - "border-y-rose-700", - "border-y-rose-800", - "border-y-rose-900", - "border-y-rose-950", - "border-s-inherit", - "border-s-current", - "border-s-transparent", - "border-s-black", - "border-s-white", - "border-s-slate-50", - "border-s-slate-100", - "border-s-slate-200", - "border-s-slate-300", - "border-s-slate-400", - "border-s-slate-500", - "border-s-slate-600", - "border-s-slate-700", - "border-s-slate-800", - "border-s-slate-900", - "border-s-slate-950", - "border-s-gray-50", - "border-s-gray-100", - "border-s-gray-200", - "border-s-gray-300", - "border-s-gray-400", - "border-s-gray-500", - "border-s-gray-600", - "border-s-gray-700", - "border-s-gray-800", - "border-s-gray-900", - "border-s-gray-950", - "border-s-zinc-50", - "border-s-zinc-100", - "border-s-zinc-200", - "border-s-zinc-300", - "border-s-zinc-400", - "border-s-zinc-500", - "border-s-zinc-600", - "border-s-zinc-700", - "border-s-zinc-800", - "border-s-zinc-900", - "border-s-zinc-950", - "border-s-neutral-50", - "border-s-neutral-100", - "border-s-neutral-200", - "border-s-neutral-300", - "border-s-neutral-400", - "border-s-neutral-500", - "border-s-neutral-600", - "border-s-neutral-700", - "border-s-neutral-800", - "border-s-neutral-900", - "border-s-neutral-950", - "border-s-stone-50", - "border-s-stone-100", - "border-s-stone-200", - "border-s-stone-300", - "border-s-stone-400", - "border-s-stone-500", - "border-s-stone-600", - "border-s-stone-700", - "border-s-stone-800", - "border-s-stone-900", - "border-s-stone-950", - "border-s-red-50", - "border-s-red-100", - "border-s-red-200", - "border-s-red-300", - "border-s-red-400", - "border-s-red-500", - "border-s-red-600", - "border-s-red-700", - "border-s-red-800", - "border-s-red-900", - "border-s-red-950", - "border-s-orange-50", - "border-s-orange-100", - "border-s-orange-200", - "border-s-orange-300", - "border-s-orange-400", - "border-s-orange-500", - "border-s-orange-600", - "border-s-orange-700", - "border-s-orange-800", - "border-s-orange-900", - "border-s-orange-950", - "border-s-amber-50", - "border-s-amber-100", - "border-s-amber-200", - "border-s-amber-300", - "border-s-amber-400", - "border-s-amber-500", - "border-s-amber-600", - "border-s-amber-700", - "border-s-amber-800", - "border-s-amber-900", - "border-s-amber-950", - "border-s-yellow-50", - "border-s-yellow-100", - "border-s-yellow-200", - "border-s-yellow-300", - "border-s-yellow-400", - "border-s-yellow-500", - "border-s-yellow-600", - "border-s-yellow-700", - "border-s-yellow-800", - "border-s-yellow-900", - "border-s-yellow-950", - "border-s-lime-50", - "border-s-lime-100", - "border-s-lime-200", - "border-s-lime-300", - "border-s-lime-400", - "border-s-lime-500", - "border-s-lime-600", - "border-s-lime-700", - "border-s-lime-800", - "border-s-lime-900", - "border-s-lime-950", - "border-s-green-50", - "border-s-green-100", - "border-s-green-200", - "border-s-green-300", - "border-s-green-400", - "border-s-green-500", - "border-s-green-600", - "border-s-green-700", - "border-s-green-800", - "border-s-green-900", - "border-s-green-950", - "border-s-emerald-50", - "border-s-emerald-100", - "border-s-emerald-200", - "border-s-emerald-300", - "border-s-emerald-400", - "border-s-emerald-500", - "border-s-emerald-600", - "border-s-emerald-700", - "border-s-emerald-800", - "border-s-emerald-900", - "border-s-emerald-950", - "border-s-teal-50", - "border-s-teal-100", - "border-s-teal-200", - "border-s-teal-300", - "border-s-teal-400", - "border-s-teal-500", - "border-s-teal-600", - "border-s-teal-700", - "border-s-teal-800", - "border-s-teal-900", - "border-s-teal-950", - "border-s-cyan-50", - "border-s-cyan-100", - "border-s-cyan-200", - "border-s-cyan-300", - "border-s-cyan-400", - "border-s-cyan-500", - "border-s-cyan-600", - "border-s-cyan-700", - "border-s-cyan-800", - "border-s-cyan-900", - "border-s-cyan-950", - "border-s-sky-50", - "border-s-sky-100", - "border-s-sky-200", - "border-s-sky-300", - "border-s-sky-400", - "border-s-sky-500", - "border-s-sky-600", - "border-s-sky-700", - "border-s-sky-800", - "border-s-sky-900", - "border-s-sky-950", - "border-s-blue-50", - "border-s-blue-100", - "border-s-blue-200", - "border-s-blue-300", - "border-s-blue-400", - "border-s-blue-500", - "border-s-blue-600", - "border-s-blue-700", - "border-s-blue-800", - "border-s-blue-900", - "border-s-blue-950", - "border-s-indigo-50", - "border-s-indigo-100", - "border-s-indigo-200", - "border-s-indigo-300", - "border-s-indigo-400", - "border-s-indigo-500", - "border-s-indigo-600", - "border-s-indigo-700", - "border-s-indigo-800", - "border-s-indigo-900", - "border-s-indigo-950", - "border-s-violet-50", - "border-s-violet-100", - "border-s-violet-200", - "border-s-violet-300", - "border-s-violet-400", - "border-s-violet-500", - "border-s-violet-600", - "border-s-violet-700", - "border-s-violet-800", - "border-s-violet-900", - "border-s-violet-950", - "border-s-purple-50", - "border-s-purple-100", - "border-s-purple-200", - "border-s-purple-300", - "border-s-purple-400", - "border-s-purple-500", - "border-s-purple-600", - "border-s-purple-700", - "border-s-purple-800", - "border-s-purple-900", - "border-s-purple-950", - "border-s-fuchsia-50", - "border-s-fuchsia-100", - "border-s-fuchsia-200", - "border-s-fuchsia-300", - "border-s-fuchsia-400", - "border-s-fuchsia-500", - "border-s-fuchsia-600", - "border-s-fuchsia-700", - "border-s-fuchsia-800", - "border-s-fuchsia-900", - "border-s-fuchsia-950", - "border-s-pink-50", - "border-s-pink-100", - "border-s-pink-200", - "border-s-pink-300", - "border-s-pink-400", - "border-s-pink-500", - "border-s-pink-600", - "border-s-pink-700", - "border-s-pink-800", - "border-s-pink-900", - "border-s-pink-950", - "border-s-rose-50", - "border-s-rose-100", - "border-s-rose-200", - "border-s-rose-300", - "border-s-rose-400", - "border-s-rose-500", - "border-s-rose-600", - "border-s-rose-700", - "border-s-rose-800", - "border-s-rose-900", - "border-s-rose-950", - "border-e-inherit", - "border-e-current", - "border-e-transparent", - "border-e-black", - "border-e-white", - "border-e-slate-50", - "border-e-slate-100", - "border-e-slate-200", - "border-e-slate-300", - "border-e-slate-400", - "border-e-slate-500", - "border-e-slate-600", - "border-e-slate-700", - "border-e-slate-800", - "border-e-slate-900", - "border-e-slate-950", - "border-e-gray-50", - "border-e-gray-100", - "border-e-gray-200", - "border-e-gray-300", - "border-e-gray-400", - "border-e-gray-500", - "border-e-gray-600", - "border-e-gray-700", - "border-e-gray-800", - "border-e-gray-900", - "border-e-gray-950", - "border-e-zinc-50", - "border-e-zinc-100", - "border-e-zinc-200", - "border-e-zinc-300", - "border-e-zinc-400", - "border-e-zinc-500", - "border-e-zinc-600", - "border-e-zinc-700", - "border-e-zinc-800", - "border-e-zinc-900", - "border-e-zinc-950", - "border-e-neutral-50", - "border-e-neutral-100", - "border-e-neutral-200", - "border-e-neutral-300", - "border-e-neutral-400", - "border-e-neutral-500", - "border-e-neutral-600", - "border-e-neutral-700", - "border-e-neutral-800", - "border-e-neutral-900", - "border-e-neutral-950", - "border-e-stone-50", - "border-e-stone-100", - "border-e-stone-200", - "border-e-stone-300", - "border-e-stone-400", - "border-e-stone-500", - "border-e-stone-600", - "border-e-stone-700", - "border-e-stone-800", - "border-e-stone-900", - "border-e-stone-950", - "border-e-red-50", - "border-e-red-100", - "border-e-red-200", - "border-e-red-300", - "border-e-red-400", - "border-e-red-500", - "border-e-red-600", - "border-e-red-700", - "border-e-red-800", - "border-e-red-900", - "border-e-red-950", - "border-e-orange-50", - "border-e-orange-100", - "border-e-orange-200", - "border-e-orange-300", - "border-e-orange-400", - "border-e-orange-500", - "border-e-orange-600", - "border-e-orange-700", - "border-e-orange-800", - "border-e-orange-900", - "border-e-orange-950", - "border-e-amber-50", - "border-e-amber-100", - "border-e-amber-200", - "border-e-amber-300", - "border-e-amber-400", - "border-e-amber-500", - "border-e-amber-600", - "border-e-amber-700", - "border-e-amber-800", - "border-e-amber-900", - "border-e-amber-950", - "border-e-yellow-50", - "border-e-yellow-100", - "border-e-yellow-200", - "border-e-yellow-300", - "border-e-yellow-400", - "border-e-yellow-500", - "border-e-yellow-600", - "border-e-yellow-700", - "border-e-yellow-800", - "border-e-yellow-900", - "border-e-yellow-950", - "border-e-lime-50", - "border-e-lime-100", - "border-e-lime-200", - "border-e-lime-300", - "border-e-lime-400", - "border-e-lime-500", - "border-e-lime-600", - "border-e-lime-700", - "border-e-lime-800", - "border-e-lime-900", - "border-e-lime-950", - "border-e-green-50", - "border-e-green-100", - "border-e-green-200", - "border-e-green-300", - "border-e-green-400", - "border-e-green-500", - "border-e-green-600", - "border-e-green-700", - "border-e-green-800", - "border-e-green-900", - "border-e-green-950", - "border-e-emerald-50", - "border-e-emerald-100", - "border-e-emerald-200", - "border-e-emerald-300", - "border-e-emerald-400", - "border-e-emerald-500", - "border-e-emerald-600", - "border-e-emerald-700", - "border-e-emerald-800", - "border-e-emerald-900", - "border-e-emerald-950", - "border-e-teal-50", - "border-e-teal-100", - "border-e-teal-200", - "border-e-teal-300", - "border-e-teal-400", - "border-e-teal-500", - "border-e-teal-600", - "border-e-teal-700", - "border-e-teal-800", - "border-e-teal-900", - "border-e-teal-950", - "border-e-cyan-50", - "border-e-cyan-100", - "border-e-cyan-200", - "border-e-cyan-300", - "border-e-cyan-400", - "border-e-cyan-500", - "border-e-cyan-600", - "border-e-cyan-700", - "border-e-cyan-800", - "border-e-cyan-900", - "border-e-cyan-950", - "border-e-sky-50", - "border-e-sky-100", - "border-e-sky-200", - "border-e-sky-300", - "border-e-sky-400", - "border-e-sky-500", - "border-e-sky-600", - "border-e-sky-700", - "border-e-sky-800", - "border-e-sky-900", - "border-e-sky-950", - "border-e-blue-50", - "border-e-blue-100", - "border-e-blue-200", - "border-e-blue-300", - "border-e-blue-400", - "border-e-blue-500", - "border-e-blue-600", - "border-e-blue-700", - "border-e-blue-800", - "border-e-blue-900", - "border-e-blue-950", - "border-e-indigo-50", - "border-e-indigo-100", - "border-e-indigo-200", - "border-e-indigo-300", - "border-e-indigo-400", - "border-e-indigo-500", - "border-e-indigo-600", - "border-e-indigo-700", - "border-e-indigo-800", - "border-e-indigo-900", - "border-e-indigo-950", - "border-e-violet-50", - "border-e-violet-100", - "border-e-violet-200", - "border-e-violet-300", - "border-e-violet-400", - "border-e-violet-500", - "border-e-violet-600", - "border-e-violet-700", - "border-e-violet-800", - "border-e-violet-900", - "border-e-violet-950", - "border-e-purple-50", - "border-e-purple-100", - "border-e-purple-200", - "border-e-purple-300", - "border-e-purple-400", - "border-e-purple-500", - "border-e-purple-600", - "border-e-purple-700", - "border-e-purple-800", - "border-e-purple-900", - "border-e-purple-950", - "border-e-fuchsia-50", - "border-e-fuchsia-100", - "border-e-fuchsia-200", - "border-e-fuchsia-300", - "border-e-fuchsia-400", - "border-e-fuchsia-500", - "border-e-fuchsia-600", - "border-e-fuchsia-700", - "border-e-fuchsia-800", - "border-e-fuchsia-900", - "border-e-fuchsia-950", - "border-e-pink-50", - "border-e-pink-100", - "border-e-pink-200", - "border-e-pink-300", - "border-e-pink-400", - "border-e-pink-500", - "border-e-pink-600", - "border-e-pink-700", - "border-e-pink-800", - "border-e-pink-900", - "border-e-pink-950", - "border-e-rose-50", - "border-e-rose-100", - "border-e-rose-200", - "border-e-rose-300", - "border-e-rose-400", - "border-e-rose-500", - "border-e-rose-600", - "border-e-rose-700", - "border-e-rose-800", - "border-e-rose-900", - "border-e-rose-950", - "border-t-inherit", - "border-t-current", - "border-t-transparent", - "border-t-black", - "border-t-white", - "border-t-slate-50", - "border-t-slate-100", - "border-t-slate-200", - "border-t-slate-300", - "border-t-slate-400", - "border-t-slate-500", - "border-t-slate-600", - "border-t-slate-700", - "border-t-slate-800", - "border-t-slate-900", - "border-t-slate-950", - "border-t-gray-50", - "border-t-gray-100", - "border-t-gray-200", - "border-t-gray-300", - "border-t-gray-400", - "border-t-gray-500", - "border-t-gray-600", - "border-t-gray-700", - "border-t-gray-800", - "border-t-gray-900", - "border-t-gray-950", - "border-t-zinc-50", - "border-t-zinc-100", - "border-t-zinc-200", - "border-t-zinc-300", - "border-t-zinc-400", - "border-t-zinc-500", - "border-t-zinc-600", - "border-t-zinc-700", - "border-t-zinc-800", - "border-t-zinc-900", - "border-t-zinc-950", - "border-t-neutral-50", - "border-t-neutral-100", - "border-t-neutral-200", - "border-t-neutral-300", - "border-t-neutral-400", - "border-t-neutral-500", - "border-t-neutral-600", - "border-t-neutral-700", - "border-t-neutral-800", - "border-t-neutral-900", - "border-t-neutral-950", - "border-t-stone-50", - "border-t-stone-100", - "border-t-stone-200", - "border-t-stone-300", - "border-t-stone-400", - "border-t-stone-500", - "border-t-stone-600", - "border-t-stone-700", - "border-t-stone-800", - "border-t-stone-900", - "border-t-stone-950", - "border-t-red-50", - "border-t-red-100", - "border-t-red-200", - "border-t-red-300", - "border-t-red-400", - "border-t-red-500", - "border-t-red-600", - "border-t-red-700", - "border-t-red-800", - "border-t-red-900", - "border-t-red-950", - "border-t-orange-50", - "border-t-orange-100", - "border-t-orange-200", - "border-t-orange-300", - "border-t-orange-400", - "border-t-orange-500", - "border-t-orange-600", - "border-t-orange-700", - "border-t-orange-800", - "border-t-orange-900", - "border-t-orange-950", - "border-t-amber-50", - "border-t-amber-100", - "border-t-amber-200", - "border-t-amber-300", - "border-t-amber-400", - "border-t-amber-500", - "border-t-amber-600", - "border-t-amber-700", - "border-t-amber-800", - "border-t-amber-900", - "border-t-amber-950", - "border-t-yellow-50", - "border-t-yellow-100", - "border-t-yellow-200", - "border-t-yellow-300", - "border-t-yellow-400", - "border-t-yellow-500", - "border-t-yellow-600", - "border-t-yellow-700", - "border-t-yellow-800", - "border-t-yellow-900", - "border-t-yellow-950", - "border-t-lime-50", - "border-t-lime-100", - "border-t-lime-200", - "border-t-lime-300", - "border-t-lime-400", - "border-t-lime-500", - "border-t-lime-600", - "border-t-lime-700", - "border-t-lime-800", - "border-t-lime-900", - "border-t-lime-950", - "border-t-green-50", - "border-t-green-100", - "border-t-green-200", - "border-t-green-300", - "border-t-green-400", - "border-t-green-500", - "border-t-green-600", - "border-t-green-700", - "border-t-green-800", - "border-t-green-900", - "border-t-green-950", - "border-t-emerald-50", - "border-t-emerald-100", - "border-t-emerald-200", - "border-t-emerald-300", - "border-t-emerald-400", - "border-t-emerald-500", - "border-t-emerald-600", - "border-t-emerald-700", - "border-t-emerald-800", - "border-t-emerald-900", - "border-t-emerald-950", - "border-t-teal-50", - "border-t-teal-100", - "border-t-teal-200", - "border-t-teal-300", - "border-t-teal-400", - "border-t-teal-500", - "border-t-teal-600", - "border-t-teal-700", - "border-t-teal-800", - "border-t-teal-900", - "border-t-teal-950", - "border-t-cyan-50", - "border-t-cyan-100", - "border-t-cyan-200", - "border-t-cyan-300", - "border-t-cyan-400", - "border-t-cyan-500", - "border-t-cyan-600", - "border-t-cyan-700", - "border-t-cyan-800", - "border-t-cyan-900", - "border-t-cyan-950", - "border-t-sky-50", - "border-t-sky-100", - "border-t-sky-200", - "border-t-sky-300", - "border-t-sky-400", - "border-t-sky-500", - "border-t-sky-600", - "border-t-sky-700", - "border-t-sky-800", - "border-t-sky-900", - "border-t-sky-950", - "border-t-blue-50", - "border-t-blue-100", - "border-t-blue-200", - "border-t-blue-300", - "border-t-blue-400", - "border-t-blue-500", - "border-t-blue-600", - "border-t-blue-700", - "border-t-blue-800", - "border-t-blue-900", - "border-t-blue-950", - "border-t-indigo-50", - "border-t-indigo-100", - "border-t-indigo-200", - "border-t-indigo-300", - "border-t-indigo-400", - "border-t-indigo-500", - "border-t-indigo-600", - "border-t-indigo-700", - "border-t-indigo-800", - "border-t-indigo-900", - "border-t-indigo-950", - "border-t-violet-50", - "border-t-violet-100", - "border-t-violet-200", - "border-t-violet-300", - "border-t-violet-400", - "border-t-violet-500", - "border-t-violet-600", - "border-t-violet-700", - "border-t-violet-800", - "border-t-violet-900", - "border-t-violet-950", - "border-t-purple-50", - "border-t-purple-100", - "border-t-purple-200", - "border-t-purple-300", - "border-t-purple-400", - "border-t-purple-500", - "border-t-purple-600", - "border-t-purple-700", - "border-t-purple-800", - "border-t-purple-900", - "border-t-purple-950", - "border-t-fuchsia-50", - "border-t-fuchsia-100", - "border-t-fuchsia-200", - "border-t-fuchsia-300", - "border-t-fuchsia-400", - "border-t-fuchsia-500", - "border-t-fuchsia-600", - "border-t-fuchsia-700", - "border-t-fuchsia-800", - "border-t-fuchsia-900", - "border-t-fuchsia-950", - "border-t-pink-50", - "border-t-pink-100", - "border-t-pink-200", - "border-t-pink-300", - "border-t-pink-400", - "border-t-pink-500", - "border-t-pink-600", - "border-t-pink-700", - "border-t-pink-800", - "border-t-pink-900", - "border-t-pink-950", - "border-t-rose-50", - "border-t-rose-100", - "border-t-rose-200", - "border-t-rose-300", - "border-t-rose-400", - "border-t-rose-500", - "border-t-rose-600", - "border-t-rose-700", - "border-t-rose-800", - "border-t-rose-900", - "border-t-rose-950", - "border-r-inherit", - "border-r-current", - "border-r-transparent", - "border-r-black", - "border-r-white", - "border-r-slate-50", - "border-r-slate-100", - "border-r-slate-200", - "border-r-slate-300", - "border-r-slate-400", - "border-r-slate-500", - "border-r-slate-600", - "border-r-slate-700", - "border-r-slate-800", - "border-r-slate-900", - "border-r-slate-950", - "border-r-gray-50", - "border-r-gray-100", - "border-r-gray-200", - "border-r-gray-300", - "border-r-gray-400", - "border-r-gray-500", - "border-r-gray-600", - "border-r-gray-700", - "border-r-gray-800", - "border-r-gray-900", - "border-r-gray-950", - "border-r-zinc-50", - "border-r-zinc-100", - "border-r-zinc-200", - "border-r-zinc-300", - "border-r-zinc-400", - "border-r-zinc-500", - "border-r-zinc-600", - "border-r-zinc-700", - "border-r-zinc-800", - "border-r-zinc-900", - "border-r-zinc-950", - "border-r-neutral-50", - "border-r-neutral-100", - "border-r-neutral-200", - "border-r-neutral-300", - "border-r-neutral-400", - "border-r-neutral-500", - "border-r-neutral-600", - "border-r-neutral-700", - "border-r-neutral-800", - "border-r-neutral-900", - "border-r-neutral-950", - "border-r-stone-50", - "border-r-stone-100", - "border-r-stone-200", - "border-r-stone-300", - "border-r-stone-400", - "border-r-stone-500", - "border-r-stone-600", - "border-r-stone-700", - "border-r-stone-800", - "border-r-stone-900", - "border-r-stone-950", - "border-r-red-50", - "border-r-red-100", - "border-r-red-200", - "border-r-red-300", - "border-r-red-400", - "border-r-red-500", - "border-r-red-600", - "border-r-red-700", - "border-r-red-800", - "border-r-red-900", - "border-r-red-950", - "border-r-orange-50", - "border-r-orange-100", - "border-r-orange-200", - "border-r-orange-300", - "border-r-orange-400", - "border-r-orange-500", - "border-r-orange-600", - "border-r-orange-700", - "border-r-orange-800", - "border-r-orange-900", - "border-r-orange-950", - "border-r-amber-50", - "border-r-amber-100", - "border-r-amber-200", - "border-r-amber-300", - "border-r-amber-400", - "border-r-amber-500", - "border-r-amber-600", - "border-r-amber-700", - "border-r-amber-800", - "border-r-amber-900", - "border-r-amber-950", - "border-r-yellow-50", - "border-r-yellow-100", - "border-r-yellow-200", - "border-r-yellow-300", - "border-r-yellow-400", - "border-r-yellow-500", - "border-r-yellow-600", - "border-r-yellow-700", - "border-r-yellow-800", - "border-r-yellow-900", - "border-r-yellow-950", - "border-r-lime-50", - "border-r-lime-100", - "border-r-lime-200", - "border-r-lime-300", - "border-r-lime-400", - "border-r-lime-500", - "border-r-lime-600", - "border-r-lime-700", - "border-r-lime-800", - "border-r-lime-900", - "border-r-lime-950", - "border-r-green-50", - "border-r-green-100", - "border-r-green-200", - "border-r-green-300", - "border-r-green-400", - "border-r-green-500", - "border-r-green-600", - "border-r-green-700", - "border-r-green-800", - "border-r-green-900", - "border-r-green-950", - "border-r-emerald-50", - "border-r-emerald-100", - "border-r-emerald-200", - "border-r-emerald-300", - "border-r-emerald-400", - "border-r-emerald-500", - "border-r-emerald-600", - "border-r-emerald-700", - "border-r-emerald-800", - "border-r-emerald-900", - "border-r-emerald-950", - "border-r-teal-50", - "border-r-teal-100", - "border-r-teal-200", - "border-r-teal-300", - "border-r-teal-400", - "border-r-teal-500", - "border-r-teal-600", - "border-r-teal-700", - "border-r-teal-800", - "border-r-teal-900", - "border-r-teal-950", - "border-r-cyan-50", - "border-r-cyan-100", - "border-r-cyan-200", - "border-r-cyan-300", - "border-r-cyan-400", - "border-r-cyan-500", - "border-r-cyan-600", - "border-r-cyan-700", - "border-r-cyan-800", - "border-r-cyan-900", - "border-r-cyan-950", - "border-r-sky-50", - "border-r-sky-100", - "border-r-sky-200", - "border-r-sky-300", - "border-r-sky-400", - "border-r-sky-500", - "border-r-sky-600", - "border-r-sky-700", - "border-r-sky-800", - "border-r-sky-900", - "border-r-sky-950", - "border-r-blue-50", - "border-r-blue-100", - "border-r-blue-200", - "border-r-blue-300", - "border-r-blue-400", - "border-r-blue-500", - "border-r-blue-600", - "border-r-blue-700", - "border-r-blue-800", - "border-r-blue-900", - "border-r-blue-950", - "border-r-indigo-50", - "border-r-indigo-100", - "border-r-indigo-200", - "border-r-indigo-300", - "border-r-indigo-400", - "border-r-indigo-500", - "border-r-indigo-600", - "border-r-indigo-700", - "border-r-indigo-800", - "border-r-indigo-900", - "border-r-indigo-950", - "border-r-violet-50", - "border-r-violet-100", - "border-r-violet-200", - "border-r-violet-300", - "border-r-violet-400", - "border-r-violet-500", - "border-r-violet-600", - "border-r-violet-700", - "border-r-violet-800", - "border-r-violet-900", - "border-r-violet-950", - "border-r-purple-50", - "border-r-purple-100", - "border-r-purple-200", - "border-r-purple-300", - "border-r-purple-400", - "border-r-purple-500", - "border-r-purple-600", - "border-r-purple-700", - "border-r-purple-800", - "border-r-purple-900", - "border-r-purple-950", - "border-r-fuchsia-50", - "border-r-fuchsia-100", - "border-r-fuchsia-200", - "border-r-fuchsia-300", - "border-r-fuchsia-400", - "border-r-fuchsia-500", - "border-r-fuchsia-600", - "border-r-fuchsia-700", - "border-r-fuchsia-800", - "border-r-fuchsia-900", - "border-r-fuchsia-950", - "border-r-pink-50", - "border-r-pink-100", - "border-r-pink-200", - "border-r-pink-300", - "border-r-pink-400", - "border-r-pink-500", - "border-r-pink-600", - "border-r-pink-700", - "border-r-pink-800", - "border-r-pink-900", - "border-r-pink-950", - "border-r-rose-50", - "border-r-rose-100", - "border-r-rose-200", - "border-r-rose-300", - "border-r-rose-400", - "border-r-rose-500", - "border-r-rose-600", - "border-r-rose-700", - "border-r-rose-800", - "border-r-rose-900", - "border-r-rose-950", - "border-b-inherit", - "border-b-current", - "border-b-transparent", - "border-b-black", - "border-b-white", - "border-b-slate-50", - "border-b-slate-100", - "border-b-slate-200", - "border-b-slate-300", - "border-b-slate-400", - "border-b-slate-500", - "border-b-slate-600", - "border-b-slate-700", - "border-b-slate-800", - "border-b-slate-900", - "border-b-slate-950", - "border-b-gray-50", - "border-b-gray-100", - "border-b-gray-200", - "border-b-gray-300", - "border-b-gray-400", - "border-b-gray-500", - "border-b-gray-600", - "border-b-gray-700", - "border-b-gray-800", - "border-b-gray-900", - "border-b-gray-950", - "border-b-zinc-50", - "border-b-zinc-100", - "border-b-zinc-200", - "border-b-zinc-300", - "border-b-zinc-400", - "border-b-zinc-500", - "border-b-zinc-600", - "border-b-zinc-700", - "border-b-zinc-800", - "border-b-zinc-900", - "border-b-zinc-950", - "border-b-neutral-50", - "border-b-neutral-100", - "border-b-neutral-200", - "border-b-neutral-300", - "border-b-neutral-400", - "border-b-neutral-500", - "border-b-neutral-600", - "border-b-neutral-700", - "border-b-neutral-800", - "border-b-neutral-900", - "border-b-neutral-950", - "border-b-stone-50", - "border-b-stone-100", - "border-b-stone-200", - "border-b-stone-300", - "border-b-stone-400", - "border-b-stone-500", - "border-b-stone-600", - "border-b-stone-700", - "border-b-stone-800", - "border-b-stone-900", - "border-b-stone-950", - "border-b-red-50", - "border-b-red-100", - "border-b-red-200", - "border-b-red-300", - "border-b-red-400", - "border-b-red-500", - "border-b-red-600", - "border-b-red-700", - "border-b-red-800", - "border-b-red-900", - "border-b-red-950", - "border-b-orange-50", - "border-b-orange-100", - "border-b-orange-200", - "border-b-orange-300", - "border-b-orange-400", - "border-b-orange-500", - "border-b-orange-600", - "border-b-orange-700", - "border-b-orange-800", - "border-b-orange-900", - "border-b-orange-950", - "border-b-amber-50", - "border-b-amber-100", - "border-b-amber-200", - "border-b-amber-300", - "border-b-amber-400", - "border-b-amber-500", - "border-b-amber-600", - "border-b-amber-700", - "border-b-amber-800", - "border-b-amber-900", - "border-b-amber-950", - "border-b-yellow-50", - "border-b-yellow-100", - "border-b-yellow-200", - "border-b-yellow-300", - "border-b-yellow-400", - "border-b-yellow-500", - "border-b-yellow-600", - "border-b-yellow-700", - "border-b-yellow-800", - "border-b-yellow-900", - "border-b-yellow-950", - "border-b-lime-50", - "border-b-lime-100", - "border-b-lime-200", - "border-b-lime-300", - "border-b-lime-400", - "border-b-lime-500", - "border-b-lime-600", - "border-b-lime-700", - "border-b-lime-800", - "border-b-lime-900", - "border-b-lime-950", - "border-b-green-50", - "border-b-green-100", - "border-b-green-200", - "border-b-green-300", - "border-b-green-400", - "border-b-green-500", - "border-b-green-600", - "border-b-green-700", - "border-b-green-800", - "border-b-green-900", - "border-b-green-950", - "border-b-emerald-50", - "border-b-emerald-100", - "border-b-emerald-200", - "border-b-emerald-300", - "border-b-emerald-400", - "border-b-emerald-500", - "border-b-emerald-600", - "border-b-emerald-700", - "border-b-emerald-800", - "border-b-emerald-900", - "border-b-emerald-950", - "border-b-teal-50", - "border-b-teal-100", - "border-b-teal-200", - "border-b-teal-300", - "border-b-teal-400", - "border-b-teal-500", - "border-b-teal-600", - "border-b-teal-700", - "border-b-teal-800", - "border-b-teal-900", - "border-b-teal-950", - "border-b-cyan-50", - "border-b-cyan-100", - "border-b-cyan-200", - "border-b-cyan-300", - "border-b-cyan-400", - "border-b-cyan-500", - "border-b-cyan-600", - "border-b-cyan-700", - "border-b-cyan-800", - "border-b-cyan-900", - "border-b-cyan-950", - "border-b-sky-50", - "border-b-sky-100", - "border-b-sky-200", - "border-b-sky-300", - "border-b-sky-400", - "border-b-sky-500", - "border-b-sky-600", - "border-b-sky-700", - "border-b-sky-800", - "border-b-sky-900", - "border-b-sky-950", - "border-b-blue-50", - "border-b-blue-100", - "border-b-blue-200", - "border-b-blue-300", - "border-b-blue-400", - "border-b-blue-500", - "border-b-blue-600", - "border-b-blue-700", - "border-b-blue-800", - "border-b-blue-900", - "border-b-blue-950", - "border-b-indigo-50", - "border-b-indigo-100", - "border-b-indigo-200", - "border-b-indigo-300", - "border-b-indigo-400", - "border-b-indigo-500", - "border-b-indigo-600", - "border-b-indigo-700", - "border-b-indigo-800", - "border-b-indigo-900", - "border-b-indigo-950", - "border-b-violet-50", - "border-b-violet-100", - "border-b-violet-200", - "border-b-violet-300", - "border-b-violet-400", - "border-b-violet-500", - "border-b-violet-600", - "border-b-violet-700", - "border-b-violet-800", - "border-b-violet-900", - "border-b-violet-950", - "border-b-purple-50", - "border-b-purple-100", - "border-b-purple-200", - "border-b-purple-300", - "border-b-purple-400", - "border-b-purple-500", - "border-b-purple-600", - "border-b-purple-700", - "border-b-purple-800", - "border-b-purple-900", - "border-b-purple-950", - "border-b-fuchsia-50", - "border-b-fuchsia-100", - "border-b-fuchsia-200", - "border-b-fuchsia-300", - "border-b-fuchsia-400", - "border-b-fuchsia-500", - "border-b-fuchsia-600", - "border-b-fuchsia-700", - "border-b-fuchsia-800", - "border-b-fuchsia-900", - "border-b-fuchsia-950", - "border-b-pink-50", - "border-b-pink-100", - "border-b-pink-200", - "border-b-pink-300", - "border-b-pink-400", - "border-b-pink-500", - "border-b-pink-600", - "border-b-pink-700", - "border-b-pink-800", - "border-b-pink-900", - "border-b-pink-950", - "border-b-rose-50", - "border-b-rose-100", - "border-b-rose-200", - "border-b-rose-300", - "border-b-rose-400", - "border-b-rose-500", - "border-b-rose-600", - "border-b-rose-700", - "border-b-rose-800", - "border-b-rose-900", - "border-b-rose-950", - "border-l-inherit", - "border-l-current", - "border-l-transparent", - "border-l-black", - "border-l-white", - "border-l-slate-50", - "border-l-slate-100", - "border-l-slate-200", - "border-l-slate-300", - "border-l-slate-400", - "border-l-slate-500", - "border-l-slate-600", - "border-l-slate-700", - "border-l-slate-800", - "border-l-slate-900", - "border-l-slate-950", - "border-l-gray-50", - "border-l-gray-100", - "border-l-gray-200", - "border-l-gray-300", - "border-l-gray-400", - "border-l-gray-500", - "border-l-gray-600", - "border-l-gray-700", - "border-l-gray-800", - "border-l-gray-900", - "border-l-gray-950", - "border-l-zinc-50", - "border-l-zinc-100", - "border-l-zinc-200", - "border-l-zinc-300", - "border-l-zinc-400", - "border-l-zinc-500", - "border-l-zinc-600", - "border-l-zinc-700", - "border-l-zinc-800", - "border-l-zinc-900", - "border-l-zinc-950", - "border-l-neutral-50", - "border-l-neutral-100", - "border-l-neutral-200", - "border-l-neutral-300", - "border-l-neutral-400", - "border-l-neutral-500", - "border-l-neutral-600", - "border-l-neutral-700", - "border-l-neutral-800", - "border-l-neutral-900", - "border-l-neutral-950", - "border-l-stone-50", - "border-l-stone-100", - "border-l-stone-200", - "border-l-stone-300", - "border-l-stone-400", - "border-l-stone-500", - "border-l-stone-600", - "border-l-stone-700", - "border-l-stone-800", - "border-l-stone-900", - "border-l-stone-950", - "border-l-red-50", - "border-l-red-100", - "border-l-red-200", - "border-l-red-300", - "border-l-red-400", - "border-l-red-500", - "border-l-red-600", - "border-l-red-700", - "border-l-red-800", - "border-l-red-900", - "border-l-red-950", - "border-l-orange-50", - "border-l-orange-100", - "border-l-orange-200", - "border-l-orange-300", - "border-l-orange-400", - "border-l-orange-500", - "border-l-orange-600", - "border-l-orange-700", - "border-l-orange-800", - "border-l-orange-900", - "border-l-orange-950", - "border-l-amber-50", - "border-l-amber-100", - "border-l-amber-200", - "border-l-amber-300", - "border-l-amber-400", - "border-l-amber-500", - "border-l-amber-600", - "border-l-amber-700", - "border-l-amber-800", - "border-l-amber-900", - "border-l-amber-950", - "border-l-yellow-50", - "border-l-yellow-100", - "border-l-yellow-200", - "border-l-yellow-300", - "border-l-yellow-400", - "border-l-yellow-500", - "border-l-yellow-600", - "border-l-yellow-700", - "border-l-yellow-800", - "border-l-yellow-900", - "border-l-yellow-950", - "border-l-lime-50", - "border-l-lime-100", - "border-l-lime-200", - "border-l-lime-300", - "border-l-lime-400", - "border-l-lime-500", - "border-l-lime-600", - "border-l-lime-700", - "border-l-lime-800", - "border-l-lime-900", - "border-l-lime-950", - "border-l-green-50", - "border-l-green-100", - "border-l-green-200", - "border-l-green-300", - "border-l-green-400", - "border-l-green-500", - "border-l-green-600", - "border-l-green-700", - "border-l-green-800", - "border-l-green-900", - "border-l-green-950", - "border-l-emerald-50", - "border-l-emerald-100", - "border-l-emerald-200", - "border-l-emerald-300", - "border-l-emerald-400", - "border-l-emerald-500", - "border-l-emerald-600", - "border-l-emerald-700", - "border-l-emerald-800", - "border-l-emerald-900", - "border-l-emerald-950", - "border-l-teal-50", - "border-l-teal-100", - "border-l-teal-200", - "border-l-teal-300", - "border-l-teal-400", - "border-l-teal-500", - "border-l-teal-600", - "border-l-teal-700", - "border-l-teal-800", - "border-l-teal-900", - "border-l-teal-950", - "border-l-cyan-50", - "border-l-cyan-100", - "border-l-cyan-200", - "border-l-cyan-300", - "border-l-cyan-400", - "border-l-cyan-500", - "border-l-cyan-600", - "border-l-cyan-700", - "border-l-cyan-800", - "border-l-cyan-900", - "border-l-cyan-950", - "border-l-sky-50", - "border-l-sky-100", - "border-l-sky-200", - "border-l-sky-300", - "border-l-sky-400", - "border-l-sky-500", - "border-l-sky-600", - "border-l-sky-700", - "border-l-sky-800", - "border-l-sky-900", - "border-l-sky-950", - "border-l-blue-50", - "border-l-blue-100", - "border-l-blue-200", - "border-l-blue-300", - "border-l-blue-400", - "border-l-blue-500", - "border-l-blue-600", - "border-l-blue-700", - "border-l-blue-800", - "border-l-blue-900", - "border-l-blue-950", - "border-l-indigo-50", - "border-l-indigo-100", - "border-l-indigo-200", - "border-l-indigo-300", - "border-l-indigo-400", - "border-l-indigo-500", - "border-l-indigo-600", - "border-l-indigo-700", - "border-l-indigo-800", - "border-l-indigo-900", - "border-l-indigo-950", - "border-l-violet-50", - "border-l-violet-100", - "border-l-violet-200", - "border-l-violet-300", - "border-l-violet-400", - "border-l-violet-500", - "border-l-violet-600", - "border-l-violet-700", - "border-l-violet-800", - "border-l-violet-900", - "border-l-violet-950", - "border-l-purple-50", - "border-l-purple-100", - "border-l-purple-200", - "border-l-purple-300", - "border-l-purple-400", - "border-l-purple-500", - "border-l-purple-600", - "border-l-purple-700", - "border-l-purple-800", - "border-l-purple-900", - "border-l-purple-950", - "border-l-fuchsia-50", - "border-l-fuchsia-100", - "border-l-fuchsia-200", - "border-l-fuchsia-300", - "border-l-fuchsia-400", - "border-l-fuchsia-500", - "border-l-fuchsia-600", - "border-l-fuchsia-700", - "border-l-fuchsia-800", - "border-l-fuchsia-900", - "border-l-fuchsia-950", - "border-l-pink-50", - "border-l-pink-100", - "border-l-pink-200", - "border-l-pink-300", - "border-l-pink-400", - "border-l-pink-500", - "border-l-pink-600", - "border-l-pink-700", - "border-l-pink-800", - "border-l-pink-900", - "border-l-pink-950", - "border-l-rose-50", - "border-l-rose-100", - "border-l-rose-200", - "border-l-rose-300", - "border-l-rose-400", - "border-l-rose-500", - "border-l-rose-600", - "border-l-rose-700", - "border-l-rose-800", - "border-l-rose-900", - "border-l-rose-950", - "border-solid", - "border-dashed", - "border-dotted", - "border-double", - "border-hidden", - "border-none", - "divide-x-0", - "divide-x-2", - "divide-x-4", - "divide-x-8", - "divide-x", - "divide-y-0", - "divide-y-2", - "divide-y-4", - "divide-y-8", - "divide-y", - "divide-y-reverse", - "divide-x-reverse", - "divide-inherit", - "divide-current", - "divide-transparent", - "divide-black", - "divide-white", - "divide-slate-50", - "divide-slate-100", - "divide-slate-200", - "divide-slate-300", - "divide-slate-400", - "divide-slate-500", - "divide-slate-600", - "divide-slate-700", - "divide-slate-800", - "divide-slate-900", - "divide-slate-950", - "divide-gray-50", - "divide-gray-100", - "divide-gray-200", - "divide-gray-300", - "divide-gray-400", - "divide-gray-500", - "divide-gray-600", - "divide-gray-700", - "divide-gray-800", - "divide-gray-900", - "divide-gray-950", - "divide-zinc-50", - "divide-zinc-100", - "divide-zinc-200", - "divide-zinc-300", - "divide-zinc-400", - "divide-zinc-500", - "divide-zinc-600", - "divide-zinc-700", - "divide-zinc-800", - "divide-zinc-900", - "divide-zinc-950", - "divide-neutral-50", - "divide-neutral-100", - "divide-neutral-200", - "divide-neutral-300", - "divide-neutral-400", - "divide-neutral-500", - "divide-neutral-600", - "divide-neutral-700", - "divide-neutral-800", - "divide-neutral-900", - "divide-neutral-950", - "divide-stone-50", - "divide-stone-100", - "divide-stone-200", - "divide-stone-300", - "divide-stone-400", - "divide-stone-500", - "divide-stone-600", - "divide-stone-700", - "divide-stone-800", - "divide-stone-900", - "divide-stone-950", - "divide-red-50", - "divide-red-100", - "divide-red-200", - "divide-red-300", - "divide-red-400", - "divide-red-500", - "divide-red-600", - "divide-red-700", - "divide-red-800", - "divide-red-900", - "divide-red-950", - "divide-orange-50", - "divide-orange-100", - "divide-orange-200", - "divide-orange-300", - "divide-orange-400", - "divide-orange-500", - "divide-orange-600", - "divide-orange-700", - "divide-orange-800", - "divide-orange-900", - "divide-orange-950", - "divide-amber-50", - "divide-amber-100", - "divide-amber-200", - "divide-amber-300", - "divide-amber-400", - "divide-amber-500", - "divide-amber-600", - "divide-amber-700", - "divide-amber-800", - "divide-amber-900", - "divide-amber-950", - "divide-yellow-50", - "divide-yellow-100", - "divide-yellow-200", - "divide-yellow-300", - "divide-yellow-400", - "divide-yellow-500", - "divide-yellow-600", - "divide-yellow-700", - "divide-yellow-800", - "divide-yellow-900", - "divide-yellow-950", - "divide-lime-50", - "divide-lime-100", - "divide-lime-200", - "divide-lime-300", - "divide-lime-400", - "divide-lime-500", - "divide-lime-600", - "divide-lime-700", - "divide-lime-800", - "divide-lime-900", - "divide-lime-950", - "divide-green-50", - "divide-green-100", - "divide-green-200", - "divide-green-300", - "divide-green-400", - "divide-green-500", - "divide-green-600", - "divide-green-700", - "divide-green-800", - "divide-green-900", - "divide-green-950", - "divide-emerald-50", - "divide-emerald-100", - "divide-emerald-200", - "divide-emerald-300", - "divide-emerald-400", - "divide-emerald-500", - "divide-emerald-600", - "divide-emerald-700", - "divide-emerald-800", - "divide-emerald-900", - "divide-emerald-950", - "divide-teal-50", - "divide-teal-100", - "divide-teal-200", - "divide-teal-300", - "divide-teal-400", - "divide-teal-500", - "divide-teal-600", - "divide-teal-700", - "divide-teal-800", - "divide-teal-900", - "divide-teal-950", - "divide-cyan-50", - "divide-cyan-100", - "divide-cyan-200", - "divide-cyan-300", - "divide-cyan-400", - "divide-cyan-500", - "divide-cyan-600", - "divide-cyan-700", - "divide-cyan-800", - "divide-cyan-900", - "divide-cyan-950", - "divide-sky-50", - "divide-sky-100", - "divide-sky-200", - "divide-sky-300", - "divide-sky-400", - "divide-sky-500", - "divide-sky-600", - "divide-sky-700", - "divide-sky-800", - "divide-sky-900", - "divide-sky-950", - "divide-blue-50", - "divide-blue-100", - "divide-blue-200", - "divide-blue-300", - "divide-blue-400", - "divide-blue-500", - "divide-blue-600", - "divide-blue-700", - "divide-blue-800", - "divide-blue-900", - "divide-blue-950", - "divide-indigo-50", - "divide-indigo-100", - "divide-indigo-200", - "divide-indigo-300", - "divide-indigo-400", - "divide-indigo-500", - "divide-indigo-600", - "divide-indigo-700", - "divide-indigo-800", - "divide-indigo-900", - "divide-indigo-950", - "divide-violet-50", - "divide-violet-100", - "divide-violet-200", - "divide-violet-300", - "divide-violet-400", - "divide-violet-500", - "divide-violet-600", - "divide-violet-700", - "divide-violet-800", - "divide-violet-900", - "divide-violet-950", - "divide-purple-50", - "divide-purple-100", - "divide-purple-200", - "divide-purple-300", - "divide-purple-400", - "divide-purple-500", - "divide-purple-600", - "divide-purple-700", - "divide-purple-800", - "divide-purple-900", - "divide-purple-950", - "divide-fuchsia-50", - "divide-fuchsia-100", - "divide-fuchsia-200", - "divide-fuchsia-300", - "divide-fuchsia-400", - "divide-fuchsia-500", - "divide-fuchsia-600", - "divide-fuchsia-700", - "divide-fuchsia-800", - "divide-fuchsia-900", - "divide-fuchsia-950", - "divide-pink-50", - "divide-pink-100", - "divide-pink-200", - "divide-pink-300", - "divide-pink-400", - "divide-pink-500", - "divide-pink-600", - "divide-pink-700", - "divide-pink-800", - "divide-pink-900", - "divide-pink-950", - "divide-rose-50", - "divide-rose-100", - "divide-rose-200", - "divide-rose-300", - "divide-rose-400", - "divide-rose-500", - "divide-rose-600", - "divide-rose-700", - "divide-rose-800", - "divide-rose-900", - "divide-rose-950", - "divide-solid", - "divide-dashed", - "divide-dotted", - "divide-double", - "divide-none", - "outline-0", - "outline-1", - "outline-2", - "outline-4", - "outline-8", - "outline-inherit", - "outline-current", - "outline-transparent", - "outline-black", - "outline-white", - "outline-slate-50", - "outline-slate-100", - "outline-slate-200", - "outline-slate-300", - "outline-slate-400", - "outline-slate-500", - "outline-slate-600", - "outline-slate-700", - "outline-slate-800", - "outline-slate-900", - "outline-slate-950", - "outline-gray-50", - "outline-gray-100", - "outline-gray-200", - "outline-gray-300", - "outline-gray-400", - "outline-gray-500", - "outline-gray-600", - "outline-gray-700", - "outline-gray-800", - "outline-gray-900", - "outline-gray-950", - "outline-zinc-50", - "outline-zinc-100", - "outline-zinc-200", - "outline-zinc-300", - "outline-zinc-400", - "outline-zinc-500", - "outline-zinc-600", - "outline-zinc-700", - "outline-zinc-800", - "outline-zinc-900", - "outline-zinc-950", - "outline-neutral-50", - "outline-neutral-100", - "outline-neutral-200", - "outline-neutral-300", - "outline-neutral-400", - "outline-neutral-500", - "outline-neutral-600", - "outline-neutral-700", - "outline-neutral-800", - "outline-neutral-900", - "outline-neutral-950", - "outline-stone-50", - "outline-stone-100", - "outline-stone-200", - "outline-stone-300", - "outline-stone-400", - "outline-stone-500", - "outline-stone-600", - "outline-stone-700", - "outline-stone-800", - "outline-stone-900", - "outline-stone-950", - "outline-red-50", - "outline-red-100", - "outline-red-200", - "outline-red-300", - "outline-red-400", - "outline-red-500", - "outline-red-600", - "outline-red-700", - "outline-red-800", - "outline-red-900", - "outline-red-950", - "outline-orange-50", - "outline-orange-100", - "outline-orange-200", - "outline-orange-300", - "outline-orange-400", - "outline-orange-500", - "outline-orange-600", - "outline-orange-700", - "outline-orange-800", - "outline-orange-900", - "outline-orange-950", - "outline-amber-50", - "outline-amber-100", - "outline-amber-200", - "outline-amber-300", - "outline-amber-400", - "outline-amber-500", - "outline-amber-600", - "outline-amber-700", - "outline-amber-800", - "outline-amber-900", - "outline-amber-950", - "outline-yellow-50", - "outline-yellow-100", - "outline-yellow-200", - "outline-yellow-300", - "outline-yellow-400", - "outline-yellow-500", - "outline-yellow-600", - "outline-yellow-700", - "outline-yellow-800", - "outline-yellow-900", - "outline-yellow-950", - "outline-lime-50", - "outline-lime-100", - "outline-lime-200", - "outline-lime-300", - "outline-lime-400", - "outline-lime-500", - "outline-lime-600", - "outline-lime-700", - "outline-lime-800", - "outline-lime-900", - "outline-lime-950", - "outline-green-50", - "outline-green-100", - "outline-green-200", - "outline-green-300", - "outline-green-400", - "outline-green-500", - "outline-green-600", - "outline-green-700", - "outline-green-800", - "outline-green-900", - "outline-green-950", - "outline-emerald-50", - "outline-emerald-100", - "outline-emerald-200", - "outline-emerald-300", - "outline-emerald-400", - "outline-emerald-500", - "outline-emerald-600", - "outline-emerald-700", - "outline-emerald-800", - "outline-emerald-900", - "outline-emerald-950", - "outline-teal-50", - "outline-teal-100", - "outline-teal-200", - "outline-teal-300", - "outline-teal-400", - "outline-teal-500", - "outline-teal-600", - "outline-teal-700", - "outline-teal-800", - "outline-teal-900", - "outline-teal-950", - "outline-cyan-50", - "outline-cyan-100", - "outline-cyan-200", - "outline-cyan-300", - "outline-cyan-400", - "outline-cyan-500", - "outline-cyan-600", - "outline-cyan-700", - "outline-cyan-800", - "outline-cyan-900", - "outline-cyan-950", - "outline-sky-50", - "outline-sky-100", - "outline-sky-200", - "outline-sky-300", - "outline-sky-400", - "outline-sky-500", - "outline-sky-600", - "outline-sky-700", - "outline-sky-800", - "outline-sky-900", - "outline-sky-950", - "outline-blue-50", - "outline-blue-100", - "outline-blue-200", - "outline-blue-300", - "outline-blue-400", - "outline-blue-500", - "outline-blue-600", - "outline-blue-700", - "outline-blue-800", - "outline-blue-900", - "outline-blue-950", - "outline-indigo-50", - "outline-indigo-100", - "outline-indigo-200", - "outline-indigo-300", - "outline-indigo-400", - "outline-indigo-500", - "outline-indigo-600", - "outline-indigo-700", - "outline-indigo-800", - "outline-indigo-900", - "outline-indigo-950", - "outline-violet-50", - "outline-violet-100", - "outline-violet-200", - "outline-violet-300", - "outline-violet-400", - "outline-violet-500", - "outline-violet-600", - "outline-violet-700", - "outline-violet-800", - "outline-violet-900", - "outline-violet-950", - "outline-purple-50", - "outline-purple-100", - "outline-purple-200", - "outline-purple-300", - "outline-purple-400", - "outline-purple-500", - "outline-purple-600", - "outline-purple-700", - "outline-purple-800", - "outline-purple-900", - "outline-purple-950", - "outline-fuchsia-50", - "outline-fuchsia-100", - "outline-fuchsia-200", - "outline-fuchsia-300", - "outline-fuchsia-400", - "outline-fuchsia-500", - "outline-fuchsia-600", - "outline-fuchsia-700", - "outline-fuchsia-800", - "outline-fuchsia-900", - "outline-fuchsia-950", - "outline-pink-50", - "outline-pink-100", - "outline-pink-200", - "outline-pink-300", - "outline-pink-400", - "outline-pink-500", - "outline-pink-600", - "outline-pink-700", - "outline-pink-800", - "outline-pink-900", - "outline-pink-950", - "outline-rose-50", - "outline-rose-100", - "outline-rose-200", - "outline-rose-300", - "outline-rose-400", - "outline-rose-500", - "outline-rose-600", - "outline-rose-700", - "outline-rose-800", - "outline-rose-900", - "outline-rose-950", - "outline-none", - "outline", - "outline-dashed", - "outline-dotted", - "outline-double", - "outline-offset-0", - "outline-offset-1", - "outline-offset-2", - "outline-offset-4", - "outline-offset-8", - "ring-0", - "ring-1", - "ring-2", - "ring", - "ring-4", - "ring-8", - "ring-inset", - "ring-inherit", - "ring-current", - "ring-transparent", - "ring-black", - "ring-white", - "ring-slate-50", - "ring-slate-100", - "ring-slate-200", - "ring-slate-300", - "ring-slate-400", - "ring-slate-500", - "ring-slate-600", - "ring-slate-700", - "ring-slate-800", - "ring-slate-900", - "ring-slate-950", - "ring-gray-50", - "ring-gray-100", - "ring-gray-200", - "ring-gray-300", - "ring-gray-400", - "ring-gray-500", - "ring-gray-600", - "ring-gray-700", - "ring-gray-800", - "ring-gray-900", - "ring-gray-950", - "ring-zinc-50", - "ring-zinc-100", - "ring-zinc-200", - "ring-zinc-300", - "ring-zinc-400", - "ring-zinc-500", - "ring-zinc-600", - "ring-zinc-700", - "ring-zinc-800", - "ring-zinc-900", - "ring-zinc-950", - "ring-neutral-50", - "ring-neutral-100", - "ring-neutral-200", - "ring-neutral-300", - "ring-neutral-400", - "ring-neutral-500", - "ring-neutral-600", - "ring-neutral-700", - "ring-neutral-800", - "ring-neutral-900", - "ring-neutral-950", - "ring-stone-50", - "ring-stone-100", - "ring-stone-200", - "ring-stone-300", - "ring-stone-400", - "ring-stone-500", - "ring-stone-600", - "ring-stone-700", - "ring-stone-800", - "ring-stone-900", - "ring-stone-950", - "ring-red-50", - "ring-red-100", - "ring-red-200", - "ring-red-300", - "ring-red-400", - "ring-red-500", - "ring-red-600", - "ring-red-700", - "ring-red-800", - "ring-red-900", - "ring-red-950", - "ring-orange-50", - "ring-orange-100", - "ring-orange-200", - "ring-orange-300", - "ring-orange-400", - "ring-orange-500", - "ring-orange-600", - "ring-orange-700", - "ring-orange-800", - "ring-orange-900", - "ring-orange-950", - "ring-amber-50", - "ring-amber-100", - "ring-amber-200", - "ring-amber-300", - "ring-amber-400", - "ring-amber-500", - "ring-amber-600", - "ring-amber-700", - "ring-amber-800", - "ring-amber-900", - "ring-amber-950", - "ring-yellow-50", - "ring-yellow-100", - "ring-yellow-200", - "ring-yellow-300", - "ring-yellow-400", - "ring-yellow-500", - "ring-yellow-600", - "ring-yellow-700", - "ring-yellow-800", - "ring-yellow-900", - "ring-yellow-950", - "ring-lime-50", - "ring-lime-100", - "ring-lime-200", - "ring-lime-300", - "ring-lime-400", - "ring-lime-500", - "ring-lime-600", - "ring-lime-700", - "ring-lime-800", - "ring-lime-900", - "ring-lime-950", - "ring-green-50", - "ring-green-100", - "ring-green-200", - "ring-green-300", - "ring-green-400", - "ring-green-500", - "ring-green-600", - "ring-green-700", - "ring-green-800", - "ring-green-900", - "ring-green-950", - "ring-emerald-50", - "ring-emerald-100", - "ring-emerald-200", - "ring-emerald-300", - "ring-emerald-400", - "ring-emerald-500", - "ring-emerald-600", - "ring-emerald-700", - "ring-emerald-800", - "ring-emerald-900", - "ring-emerald-950", - "ring-teal-50", - "ring-teal-100", - "ring-teal-200", - "ring-teal-300", - "ring-teal-400", - "ring-teal-500", - "ring-teal-600", - "ring-teal-700", - "ring-teal-800", - "ring-teal-900", - "ring-teal-950", - "ring-cyan-50", - "ring-cyan-100", - "ring-cyan-200", - "ring-cyan-300", - "ring-cyan-400", - "ring-cyan-500", - "ring-cyan-600", - "ring-cyan-700", - "ring-cyan-800", - "ring-cyan-900", - "ring-cyan-950", - "ring-sky-50", - "ring-sky-100", - "ring-sky-200", - "ring-sky-300", - "ring-sky-400", - "ring-sky-500", - "ring-sky-600", - "ring-sky-700", - "ring-sky-800", - "ring-sky-900", - "ring-sky-950", - "ring-blue-50", - "ring-blue-100", - "ring-blue-200", - "ring-blue-300", - "ring-blue-400", - "ring-blue-500", - "ring-blue-600", - "ring-blue-700", - "ring-blue-800", - "ring-blue-900", - "ring-blue-950", - "ring-indigo-50", - "ring-indigo-100", - "ring-indigo-200", - "ring-indigo-300", - "ring-indigo-400", - "ring-indigo-500", - "ring-indigo-600", - "ring-indigo-700", - "ring-indigo-800", - "ring-indigo-900", - "ring-indigo-950", - "ring-violet-50", - "ring-violet-100", - "ring-violet-200", - "ring-violet-300", - "ring-violet-400", - "ring-violet-500", - "ring-violet-600", - "ring-violet-700", - "ring-violet-800", - "ring-violet-900", - "ring-violet-950", - "ring-purple-50", - "ring-purple-100", - "ring-purple-200", - "ring-purple-300", - "ring-purple-400", - "ring-purple-500", - "ring-purple-600", - "ring-purple-700", - "ring-purple-800", - "ring-purple-900", - "ring-purple-950", - "ring-fuchsia-50", - "ring-fuchsia-100", - "ring-fuchsia-200", - "ring-fuchsia-300", - "ring-fuchsia-400", - "ring-fuchsia-500", - "ring-fuchsia-600", - "ring-fuchsia-700", - "ring-fuchsia-800", - "ring-fuchsia-900", - "ring-fuchsia-950", - "ring-pink-50", - "ring-pink-100", - "ring-pink-200", - "ring-pink-300", - "ring-pink-400", - "ring-pink-500", - "ring-pink-600", - "ring-pink-700", - "ring-pink-800", - "ring-pink-900", - "ring-pink-950", - "ring-rose-50", - "ring-rose-100", - "ring-rose-200", - "ring-rose-300", - "ring-rose-400", - "ring-rose-500", - "ring-rose-600", - "ring-rose-700", - "ring-rose-800", - "ring-rose-900", - "ring-rose-950", - "ring-offset-0", - "ring-offset-1", - "ring-offset-2", - "ring-offset-4", - "ring-offset-8", - "ring-offset-inherit", - "ring-offset-current", - "ring-offset-transparent", - "ring-offset-black", - "ring-offset-white", - "ring-offset-slate-50", - "ring-offset-slate-100", - "ring-offset-slate-200", - "ring-offset-slate-300", - "ring-offset-slate-400", - "ring-offset-slate-500", - "ring-offset-slate-600", - "ring-offset-slate-700", - "ring-offset-slate-800", - "ring-offset-slate-900", - "ring-offset-slate-950", - "ring-offset-gray-50", - "ring-offset-gray-100", - "ring-offset-gray-200", - "ring-offset-gray-300", - "ring-offset-gray-400", - "ring-offset-gray-500", - "ring-offset-gray-600", - "ring-offset-gray-700", - "ring-offset-gray-800", - "ring-offset-gray-900", - "ring-offset-gray-950", - "ring-offset-zinc-50", - "ring-offset-zinc-100", - "ring-offset-zinc-200", - "ring-offset-zinc-300", - "ring-offset-zinc-400", - "ring-offset-zinc-500", - "ring-offset-zinc-600", - "ring-offset-zinc-700", - "ring-offset-zinc-800", - "ring-offset-zinc-900", - "ring-offset-zinc-950", - "ring-offset-neutral-50", - "ring-offset-neutral-100", - "ring-offset-neutral-200", - "ring-offset-neutral-300", - "ring-offset-neutral-400", - "ring-offset-neutral-500", - "ring-offset-neutral-600", - "ring-offset-neutral-700", - "ring-offset-neutral-800", - "ring-offset-neutral-900", - "ring-offset-neutral-950", - "ring-offset-stone-50", - "ring-offset-stone-100", - "ring-offset-stone-200", - "ring-offset-stone-300", - "ring-offset-stone-400", - "ring-offset-stone-500", - "ring-offset-stone-600", - "ring-offset-stone-700", - "ring-offset-stone-800", - "ring-offset-stone-900", - "ring-offset-stone-950", - "ring-offset-red-50", - "ring-offset-red-100", - "ring-offset-red-200", - "ring-offset-red-300", - "ring-offset-red-400", - "ring-offset-red-500", - "ring-offset-red-600", - "ring-offset-red-700", - "ring-offset-red-800", - "ring-offset-red-900", - "ring-offset-red-950", - "ring-offset-orange-50", - "ring-offset-orange-100", - "ring-offset-orange-200", - "ring-offset-orange-300", - "ring-offset-orange-400", - "ring-offset-orange-500", - "ring-offset-orange-600", - "ring-offset-orange-700", - "ring-offset-orange-800", - "ring-offset-orange-900", - "ring-offset-orange-950", - "ring-offset-amber-50", - "ring-offset-amber-100", - "ring-offset-amber-200", - "ring-offset-amber-300", - "ring-offset-amber-400", - "ring-offset-amber-500", - "ring-offset-amber-600", - "ring-offset-amber-700", - "ring-offset-amber-800", - "ring-offset-amber-900", - "ring-offset-amber-950", - "ring-offset-yellow-50", - "ring-offset-yellow-100", - "ring-offset-yellow-200", - "ring-offset-yellow-300", - "ring-offset-yellow-400", - "ring-offset-yellow-500", - "ring-offset-yellow-600", - "ring-offset-yellow-700", - "ring-offset-yellow-800", - "ring-offset-yellow-900", - "ring-offset-yellow-950", - "ring-offset-lime-50", - "ring-offset-lime-100", - "ring-offset-lime-200", - "ring-offset-lime-300", - "ring-offset-lime-400", - "ring-offset-lime-500", - "ring-offset-lime-600", - "ring-offset-lime-700", - "ring-offset-lime-800", - "ring-offset-lime-900", - "ring-offset-lime-950", - "ring-offset-green-50", - "ring-offset-green-100", - "ring-offset-green-200", - "ring-offset-green-300", - "ring-offset-green-400", - "ring-offset-green-500", - "ring-offset-green-600", - "ring-offset-green-700", - "ring-offset-green-800", - "ring-offset-green-900", - "ring-offset-green-950", - "ring-offset-emerald-50", - "ring-offset-emerald-100", - "ring-offset-emerald-200", - "ring-offset-emerald-300", - "ring-offset-emerald-400", - "ring-offset-emerald-500", - "ring-offset-emerald-600", - "ring-offset-emerald-700", - "ring-offset-emerald-800", - "ring-offset-emerald-900", - "ring-offset-emerald-950", - "ring-offset-teal-50", - "ring-offset-teal-100", - "ring-offset-teal-200", - "ring-offset-teal-300", - "ring-offset-teal-400", - "ring-offset-teal-500", - "ring-offset-teal-600", - "ring-offset-teal-700", - "ring-offset-teal-800", - "ring-offset-teal-900", - "ring-offset-teal-950", - "ring-offset-cyan-50", - "ring-offset-cyan-100", - "ring-offset-cyan-200", - "ring-offset-cyan-300", - "ring-offset-cyan-400", - "ring-offset-cyan-500", - "ring-offset-cyan-600", - "ring-offset-cyan-700", - "ring-offset-cyan-800", - "ring-offset-cyan-900", - "ring-offset-cyan-950", - "ring-offset-sky-50", - "ring-offset-sky-100", - "ring-offset-sky-200", - "ring-offset-sky-300", - "ring-offset-sky-400", - "ring-offset-sky-500", - "ring-offset-sky-600", - "ring-offset-sky-700", - "ring-offset-sky-800", - "ring-offset-sky-900", - "ring-offset-sky-950", - "ring-offset-blue-50", - "ring-offset-blue-100", - "ring-offset-blue-200", - "ring-offset-blue-300", - "ring-offset-blue-400", - "ring-offset-blue-500", - "ring-offset-blue-600", - "ring-offset-blue-700", - "ring-offset-blue-800", - "ring-offset-blue-900", - "ring-offset-blue-950", - "ring-offset-indigo-50", - "ring-offset-indigo-100", - "ring-offset-indigo-200", - "ring-offset-indigo-300", - "ring-offset-indigo-400", - "ring-offset-indigo-500", - "ring-offset-indigo-600", - "ring-offset-indigo-700", - "ring-offset-indigo-800", - "ring-offset-indigo-900", - "ring-offset-indigo-950", - "ring-offset-violet-50", - "ring-offset-violet-100", - "ring-offset-violet-200", - "ring-offset-violet-300", - "ring-offset-violet-400", - "ring-offset-violet-500", - "ring-offset-violet-600", - "ring-offset-violet-700", - "ring-offset-violet-800", - "ring-offset-violet-900", - "ring-offset-violet-950", - "ring-offset-purple-50", - "ring-offset-purple-100", - "ring-offset-purple-200", - "ring-offset-purple-300", - "ring-offset-purple-400", - "ring-offset-purple-500", - "ring-offset-purple-600", - "ring-offset-purple-700", - "ring-offset-purple-800", - "ring-offset-purple-900", - "ring-offset-purple-950", - "ring-offset-fuchsia-50", - "ring-offset-fuchsia-100", - "ring-offset-fuchsia-200", - "ring-offset-fuchsia-300", - "ring-offset-fuchsia-400", - "ring-offset-fuchsia-500", - "ring-offset-fuchsia-600", - "ring-offset-fuchsia-700", - "ring-offset-fuchsia-800", - "ring-offset-fuchsia-900", - "ring-offset-fuchsia-950", - "ring-offset-pink-50", - "ring-offset-pink-100", - "ring-offset-pink-200", - "ring-offset-pink-300", - "ring-offset-pink-400", - "ring-offset-pink-500", - "ring-offset-pink-600", - "ring-offset-pink-700", - "ring-offset-pink-800", - "ring-offset-pink-900", - "ring-offset-pink-950", - "ring-offset-rose-50", - "ring-offset-rose-100", - "ring-offset-rose-200", - "ring-offset-rose-300", - "ring-offset-rose-400", - "ring-offset-rose-500", - "ring-offset-rose-600", - "ring-offset-rose-700", - "ring-offset-rose-800", - "ring-offset-rose-900", - "ring-offset-rose-950", - "shadow-sm", - "shadow", - "shadow-md", - "shadow-lg", - "shadow-xl", - "shadow-2xl", - "shadow-inner", - "shadow-none", - "shadow-inherit", - "shadow-current", - "shadow-transparent", - "shadow-black", - "shadow-white", - "shadow-slate-50", - "shadow-slate-100", - "shadow-slate-200", - "shadow-slate-300", - "shadow-slate-400", - "shadow-slate-500", - "shadow-slate-600", - "shadow-slate-700", - "shadow-slate-800", - "shadow-slate-900", - "shadow-slate-950", - "shadow-gray-50", - "shadow-gray-100", - "shadow-gray-200", - "shadow-gray-300", - "shadow-gray-400", - "shadow-gray-500", - "shadow-gray-600", - "shadow-gray-700", - "shadow-gray-800", - "shadow-gray-900", - "shadow-gray-950", - "shadow-zinc-50", - "shadow-zinc-100", - "shadow-zinc-200", - "shadow-zinc-300", - "shadow-zinc-400", - "shadow-zinc-500", - "shadow-zinc-600", - "shadow-zinc-700", - "shadow-zinc-800", - "shadow-zinc-900", - "shadow-zinc-950", - "shadow-neutral-50", - "shadow-neutral-100", - "shadow-neutral-200", - "shadow-neutral-300", - "shadow-neutral-400", - "shadow-neutral-500", - "shadow-neutral-600", - "shadow-neutral-700", - "shadow-neutral-800", - "shadow-neutral-900", - "shadow-neutral-950", - "shadow-stone-50", - "shadow-stone-100", - "shadow-stone-200", - "shadow-stone-300", - "shadow-stone-400", - "shadow-stone-500", - "shadow-stone-600", - "shadow-stone-700", - "shadow-stone-800", - "shadow-stone-900", - "shadow-stone-950", - "shadow-red-50", - "shadow-red-100", - "shadow-red-200", - "shadow-red-300", - "shadow-red-400", - "shadow-red-500", - "shadow-red-600", - "shadow-red-700", - "shadow-red-800", - "shadow-red-900", - "shadow-red-950", - "shadow-orange-50", - "shadow-orange-100", - "shadow-orange-200", - "shadow-orange-300", - "shadow-orange-400", - "shadow-orange-500", - "shadow-orange-600", - "shadow-orange-700", - "shadow-orange-800", - "shadow-orange-900", - "shadow-orange-950", - "shadow-amber-50", - "shadow-amber-100", - "shadow-amber-200", - "shadow-amber-300", - "shadow-amber-400", - "shadow-amber-500", - "shadow-amber-600", - "shadow-amber-700", - "shadow-amber-800", - "shadow-amber-900", - "shadow-amber-950", - "shadow-yellow-50", - "shadow-yellow-100", - "shadow-yellow-200", - "shadow-yellow-300", - "shadow-yellow-400", - "shadow-yellow-500", - "shadow-yellow-600", - "shadow-yellow-700", - "shadow-yellow-800", - "shadow-yellow-900", - "shadow-yellow-950", - "shadow-lime-50", - "shadow-lime-100", - "shadow-lime-200", - "shadow-lime-300", - "shadow-lime-400", - "shadow-lime-500", - "shadow-lime-600", - "shadow-lime-700", - "shadow-lime-800", - "shadow-lime-900", - "shadow-lime-950", - "shadow-green-50", - "shadow-green-100", - "shadow-green-200", - "shadow-green-300", - "shadow-green-400", - "shadow-green-500", - "shadow-green-600", - "shadow-green-700", - "shadow-green-800", - "shadow-green-900", - "shadow-green-950", - "shadow-emerald-50", - "shadow-emerald-100", - "shadow-emerald-200", - "shadow-emerald-300", - "shadow-emerald-400", - "shadow-emerald-500", - "shadow-emerald-600", - "shadow-emerald-700", - "shadow-emerald-800", - "shadow-emerald-900", - "shadow-emerald-950", - "shadow-teal-50", - "shadow-teal-100", - "shadow-teal-200", - "shadow-teal-300", - "shadow-teal-400", - "shadow-teal-500", - "shadow-teal-600", - "shadow-teal-700", - "shadow-teal-800", - "shadow-teal-900", - "shadow-teal-950", - "shadow-cyan-50", - "shadow-cyan-100", - "shadow-cyan-200", - "shadow-cyan-300", - "shadow-cyan-400", - "shadow-cyan-500", - "shadow-cyan-600", - "shadow-cyan-700", - "shadow-cyan-800", - "shadow-cyan-900", - "shadow-cyan-950", - "shadow-sky-50", - "shadow-sky-100", - "shadow-sky-200", - "shadow-sky-300", - "shadow-sky-400", - "shadow-sky-500", - "shadow-sky-600", - "shadow-sky-700", - "shadow-sky-800", - "shadow-sky-900", - "shadow-sky-950", - "shadow-blue-50", - "shadow-blue-100", - "shadow-blue-200", - "shadow-blue-300", - "shadow-blue-400", - "shadow-blue-500", - "shadow-blue-600", - "shadow-blue-700", - "shadow-blue-800", - "shadow-blue-900", - "shadow-blue-950", - "shadow-indigo-50", - "shadow-indigo-100", - "shadow-indigo-200", - "shadow-indigo-300", - "shadow-indigo-400", - "shadow-indigo-500", - "shadow-indigo-600", - "shadow-indigo-700", - "shadow-indigo-800", - "shadow-indigo-900", - "shadow-indigo-950", - "shadow-violet-50", - "shadow-violet-100", - "shadow-violet-200", - "shadow-violet-300", - "shadow-violet-400", - "shadow-violet-500", - "shadow-violet-600", - "shadow-violet-700", - "shadow-violet-800", - "shadow-violet-900", - "shadow-violet-950", - "shadow-purple-50", - "shadow-purple-100", - "shadow-purple-200", - "shadow-purple-300", - "shadow-purple-400", - "shadow-purple-500", - "shadow-purple-600", - "shadow-purple-700", - "shadow-purple-800", - "shadow-purple-900", - "shadow-purple-950", - "shadow-fuchsia-50", - "shadow-fuchsia-100", - "shadow-fuchsia-200", - "shadow-fuchsia-300", - "shadow-fuchsia-400", - "shadow-fuchsia-500", - "shadow-fuchsia-600", - "shadow-fuchsia-700", - "shadow-fuchsia-800", - "shadow-fuchsia-900", - "shadow-fuchsia-950", - "shadow-pink-50", - "shadow-pink-100", - "shadow-pink-200", - "shadow-pink-300", - "shadow-pink-400", - "shadow-pink-500", - "shadow-pink-600", - "shadow-pink-700", - "shadow-pink-800", - "shadow-pink-900", - "shadow-pink-950", - "shadow-rose-50", - "shadow-rose-100", - "shadow-rose-200", - "shadow-rose-300", - "shadow-rose-400", - "shadow-rose-500", - "shadow-rose-600", - "shadow-rose-700", - "shadow-rose-800", - "shadow-rose-900", - "shadow-rose-950", - "opacity-0", - "opacity-5", - "opacity-10", - "opacity-15", - "opacity-20", - "opacity-25", - "opacity-30", - "opacity-35", - "opacity-40", - "opacity-45", - "opacity-50", - "opacity-55", - "opacity-60", - "opacity-65", - "opacity-70", - "opacity-75", - "opacity-80", - "opacity-85", - "opacity-90", - "opacity-95", - "opacity-100", - "mix-blend-normal", - "mix-blend-multiply", - "mix-blend-screen", - "mix-blend-overlay", - "mix-blend-darken", - "mix-blend-lighten", - "mix-blend-color-dodge", - "mix-blend-color-burn", - "mix-blend-hard-light", - "mix-blend-soft-light", - "mix-blend-difference", - "mix-blend-exclusion", - "mix-blend-hue", - "mix-blend-saturation", - "mix-blend-color", - "mix-blend-luminosity", - "mix-blend-plus-darker", - "mix-blend-plus-lighter", - "bg-blend-normal", - "bg-blend-multiply", - "bg-blend-screen", - "bg-blend-overlay", - "bg-blend-darken", - "bg-blend-lighten", - "bg-blend-color-dodge", - "bg-blend-color-burn", - "bg-blend-hard-light", - "bg-blend-soft-light", - "bg-blend-difference", - "bg-blend-exclusion", - "bg-blend-hue", - "bg-blend-saturation", - "bg-blend-color", - "bg-blend-luminosity", - "blur-none", - "blur-sm", - "blur", - "blur-md", - "blur-lg", - "blur-xl", - "blur-2xl", - "blur-3xl", - "brightness-0", - "brightness-50", - "brightness-75", - "brightness-90", - "brightness-95", - "brightness-100", - "brightness-105", - "brightness-110", - "brightness-125", - "brightness-150", - "brightness-200", - "contrast-0", - "contrast-50", - "contrast-75", - "contrast-100", - "contrast-125", - "contrast-150", - "contrast-200", - "drop-shadow-sm", - "drop-shadow", - "drop-shadow-md", - "drop-shadow-lg", - "drop-shadow-xl", - "drop-shadow-2xl", - "drop-shadow-none", - "grayscale-0", - "grayscale", - "hue-rotate-0", - "hue-rotate-15", - "hue-rotate-30", - "hue-rotate-60", - "hue-rotate-90", - "hue-rotate-180", - "invert-0", - "invert", - "saturate-0", - "saturate-50", - "saturate-100", - "saturate-150", - "saturate-200", - "sepia-0", - "sepia", - "backdrop-blur-none", - "backdrop-blur-sm", - "backdrop-blur", - "backdrop-blur-md", - "backdrop-blur-lg", - "backdrop-blur-xl", - "backdrop-blur-2xl", - "backdrop-blur-3xl", - "backdrop-brightness-0", - "backdrop-brightness-50", - "backdrop-brightness-75", - "backdrop-brightness-90", - "backdrop-brightness-95", - "backdrop-brightness-100", - "backdrop-brightness-105", - "backdrop-brightness-110", - "backdrop-brightness-125", - "backdrop-brightness-150", - "backdrop-brightness-200", - "backdrop-contrast-0", - "backdrop-contrast-50", - "backdrop-contrast-75", - "backdrop-contrast-100", - "backdrop-contrast-125", - "backdrop-contrast-150", - "backdrop-contrast-200", - "backdrop-grayscale-0", - "backdrop-grayscale", - "backdrop-hue-rotate-0", - "backdrop-hue-rotate-15", - "backdrop-hue-rotate-30", - "backdrop-hue-rotate-60", - "backdrop-hue-rotate-90", - "backdrop-hue-rotate-180", - "backdrop-invert-0", - "backdrop-invert", - "backdrop-opacity-0", - "backdrop-opacity-5", - "backdrop-opacity-10", - "backdrop-opacity-15", - "backdrop-opacity-20", - "backdrop-opacity-25", - "backdrop-opacity-30", - "backdrop-opacity-35", - "backdrop-opacity-40", - "backdrop-opacity-45", - "backdrop-opacity-50", - "backdrop-opacity-55", - "backdrop-opacity-60", - "backdrop-opacity-65", - "backdrop-opacity-70", - "backdrop-opacity-75", - "backdrop-opacity-80", - "backdrop-opacity-85", - "backdrop-opacity-90", - "backdrop-opacity-95", - "backdrop-opacity-100", - "backdrop-saturate-0", - "backdrop-saturate-50", - "backdrop-saturate-100", - "backdrop-saturate-150", - "backdrop-saturate-200", - "backdrop-sepia-0", - "backdrop-sepia", - "border-collapse", - "border-separate", - "Indiana", - "Ohio", - "Michigan", - "Indiana", - "Ohio", - "Michigan", - "border-spacing-0", - "border-spacing-x-0", - "border-spacing-y-0", - "border-spacing-px", - "border-spacing-x-px", - "border-spacing-y-px", - "border-spacing-0.5", - "border-spacing-x-0.5", - "border-spacing-y-0.5", - "border-spacing-1", - "border-spacing-x-1", - "border-spacing-y-1", - "border-spacing-1.5", - "border-spacing-x-1.5", - "border-spacing-y-1.5", - "border-spacing-2", - "border-spacing-x-2", - "border-spacing-y-2", - "border-spacing-2.5", - "border-spacing-x-2.5", - "border-spacing-y-2.5", - "border-spacing-3", - "border-spacing-x-3", - "border-spacing-y-3", - "border-spacing-3.5", - "border-spacing-x-3.5", - "border-spacing-y-3.5", - "border-spacing-4", - "border-spacing-x-4", - "border-spacing-y-4", - "border-spacing-5", - "border-spacing-x-5", - "border-spacing-y-5", - "border-spacing-6", - "border-spacing-x-6", - "border-spacing-y-6", - "border-spacing-7", - "border-spacing-x-7", - "border-spacing-y-7", - "border-spacing-8", - "border-spacing-x-8", - "border-spacing-y-8", - "border-spacing-9", - "border-spacing-x-9", - "border-spacing-y-9", - "border-spacing-10", - "border-spacing-x-10", - "border-spacing-y-10", - "border-spacing-11", - "border-spacing-x-11", - "border-spacing-y-11", - "border-spacing-12", - "border-spacing-x-12", - "border-spacing-y-12", - "border-spacing-14", - "border-spacing-x-14", - "border-spacing-y-14", - "border-spacing-16", - "border-spacing-x-16", - "border-spacing-y-16", - "border-spacing-20", - "border-spacing-x-20", - "border-spacing-y-20", - "border-spacing-24", - "border-spacing-x-24", - "border-spacing-y-24", - "border-spacing-28", - "border-spacing-x-28", - "border-spacing-y-28", - "border-spacing-32", - "border-spacing-x-32", - "border-spacing-y-32", - "border-spacing-36", - "border-spacing-x-36", - "border-spacing-y-36", - "border-spacing-40", - "border-spacing-x-40", - "border-spacing-y-40", - "border-spacing-44", - "border-spacing-x-44", - "border-spacing-y-44", - "border-spacing-48", - "border-spacing-x-48", - "border-spacing-y-48", - "border-spacing-52", - "border-spacing-x-52", - "border-spacing-y-52", - "border-spacing-56", - "border-spacing-x-56", - "border-spacing-y-56", - "border-spacing-60", - "border-spacing-x-60", - "border-spacing-y-60", - "border-spacing-64", - "border-spacing-x-64", - "border-spacing-y-64", - "border-spacing-72", - "border-spacing-x-72", - "border-spacing-y-72", - "border-spacing-80", - "border-spacing-x-80", - "border-spacing-y-80", - "border-spacing-96", - "border-spacing-x-96", - "border-spacing-y-96", - "Indiana", - "Ohio", - "Michigan", - "table-auto", - "table-fixed", - "caption-top", - "caption-bottom", - "transition-none", - "transition-all", - "transition", - "transition-colors", - "transition-opacity", - "transition-shadow", - "transition-transform", - "duration-0", - "duration-75", - "duration-100", - "duration-150", - "duration-200", - "duration-300", - "duration-500", - "duration-700", - "duration-1000", - "ease-linear", - "ease-in", - "ease-out", - "ease-in-out", - "delay-0", - "delay-75", - "delay-100", - "delay-150", - "delay-200", - "delay-300", - "delay-500", - "delay-700", - "delay-1000", - "animate-none", - "animate-spin", - "animate-ping", - "animate-pulse", - "animate-bounce", - "scale-0", - "scale-x-0", - "scale-y-0", - "scale-50", - "scale-x-50", - "scale-y-50", - "scale-75", - "scale-x-75", - "scale-y-75", - "scale-90", - "scale-x-90", - "scale-y-90", - "scale-95", - "scale-x-95", - "scale-y-95", - "scale-100", - "scale-x-100", - "scale-y-100", - "scale-105", - "scale-x-105", - "scale-y-105", - "scale-110", - "scale-x-110", - "scale-y-110", - "scale-125", - "scale-x-125", - "scale-y-125", - "scale-150", - "scale-x-150", - "scale-y-150", - "rotate-0", - "rotate-1", - "rotate-2", - "rotate-3", - "rotate-6", - "rotate-12", - "rotate-45", - "rotate-90", - "rotate-180", - "translate-x-0", - "translate-y-0", - "translate-x-px", - "translate-y-px", - "translate-x-0.5", - "translate-y-0.5", - "translate-x-1", - "translate-y-1", - "translate-x-1.5", - "translate-y-1.5", - "translate-x-2", - "translate-y-2", - "translate-x-2.5", - "translate-y-2.5", - "translate-x-3", - "translate-y-3", - "translate-x-3.5", - "translate-y-3.5", - "translate-x-4", - "translate-y-4", - "translate-x-5", - "translate-y-5", - "translate-x-6", - "translate-y-6", - "translate-x-7", - "translate-y-7", - "translate-x-8", - "translate-y-8", - "translate-x-9", - "translate-y-9", - "translate-x-10", - "translate-y-10", - "translate-x-11", - "translate-y-11", - "translate-x-12", - "translate-y-12", - "translate-x-14", - "translate-y-14", - "translate-x-16", - "translate-y-16", - "translate-x-20", - "translate-y-20", - "translate-x-24", - "translate-y-24", - "translate-x-28", - "translate-y-28", - "translate-x-32", - "translate-y-32", - "translate-x-36", - "translate-y-36", - "translate-x-40", - "translate-y-40", - "translate-x-44", - "translate-y-44", - "translate-x-48", - "translate-y-48", - "translate-x-52", - "translate-y-52", - "translate-x-56", - "translate-y-56", - "translate-x-60", - "translate-y-60", - "translate-x-64", - "translate-y-64", - "translate-x-72", - "translate-y-72", - "translate-x-80", - "translate-y-80", - "translate-x-96", - "translate-y-96", - "translate-x-1/2", - "translate-x-1/3", - "translate-x-2/3", - "translate-x-1/4", - "translate-x-2/4", - "translate-x-3/4", - "translate-x-full", - "translate-y-1/2", - "translate-y-1/3", - "translate-y-2/3", - "translate-y-1/4", - "translate-y-2/4", - "translate-y-3/4", - "translate-y-full", - "skew-x-0", - "skew-y-0", - "skew-x-1", - "skew-y-1", - "skew-x-2", - "skew-y-2", - "skew-x-3", - "skew-y-3", - "skew-x-6", - "skew-y-6", - "skew-x-12", - "skew-y-12", - "origin-center", - "origin-top", - "origin-top-right", - "origin-right", - "origin-bottom-right", - "origin-bottom", - "origin-bottom-left", - "origin-left", - "origin-top-left", - "accent-inherit", - "accent-current", - "accent-transparent", - "accent-black", - "accent-white", - "accent-slate-50", - "accent-slate-100", - "accent-slate-200", - "accent-slate-300", - "accent-slate-400", - "accent-slate-500", - "accent-slate-600", - "accent-slate-700", - "accent-slate-800", - "accent-slate-900", - "accent-slate-950", - "accent-gray-50", - "accent-gray-100", - "accent-gray-200", - "accent-gray-300", - "accent-gray-400", - "accent-gray-500", - "accent-gray-600", - "accent-gray-700", - "accent-gray-800", - "accent-gray-900", - "accent-gray-950", - "accent-zinc-50", - "accent-zinc-100", - "accent-zinc-200", - "accent-zinc-300", - "accent-zinc-400", - "accent-zinc-500", - "accent-zinc-600", - "accent-zinc-700", - "accent-zinc-800", - "accent-zinc-900", - "accent-zinc-950", - "accent-neutral-50", - "accent-neutral-100", - "accent-neutral-200", - "accent-neutral-300", - "accent-neutral-400", - "accent-neutral-500", - "accent-neutral-600", - "accent-neutral-700", - "accent-neutral-800", - "accent-neutral-900", - "accent-neutral-950", - "accent-stone-50", - "accent-stone-100", - "accent-stone-200", - "accent-stone-300", - "accent-stone-400", - "accent-stone-500", - "accent-stone-600", - "accent-stone-700", - "accent-stone-800", - "accent-stone-900", - "accent-stone-950", - "accent-red-50", - "accent-red-100", - "accent-red-200", - "accent-red-300", - "accent-red-400", - "accent-red-500", - "accent-red-600", - "accent-red-700", - "accent-red-800", - "accent-red-900", - "accent-red-950", - "accent-orange-50", - "accent-orange-100", - "accent-orange-200", - "accent-orange-300", - "accent-orange-400", - "accent-orange-500", - "accent-orange-600", - "accent-orange-700", - "accent-orange-800", - "accent-orange-900", - "accent-orange-950", - "accent-amber-50", - "accent-amber-100", - "accent-amber-200", - "accent-amber-300", - "accent-amber-400", - "accent-amber-500", - "accent-amber-600", - "accent-amber-700", - "accent-amber-800", - "accent-amber-900", - "accent-amber-950", - "accent-yellow-50", - "accent-yellow-100", - "accent-yellow-200", - "accent-yellow-300", - "accent-yellow-400", - "accent-yellow-500", - "accent-yellow-600", - "accent-yellow-700", - "accent-yellow-800", - "accent-yellow-900", - "accent-yellow-950", - "accent-lime-50", - "accent-lime-100", - "accent-lime-200", - "accent-lime-300", - "accent-lime-400", - "accent-lime-500", - "accent-lime-600", - "accent-lime-700", - "accent-lime-800", - "accent-lime-900", - "accent-lime-950", - "accent-green-50", - "accent-green-100", - "accent-green-200", - "accent-green-300", - "accent-green-400", - "accent-green-500", - "accent-green-600", - "accent-green-700", - "accent-green-800", - "accent-green-900", - "accent-green-950", - "accent-emerald-50", - "accent-emerald-100", - "accent-emerald-200", - "accent-emerald-300", - "accent-emerald-400", - "accent-emerald-500", - "accent-emerald-600", - "accent-emerald-700", - "accent-emerald-800", - "accent-emerald-900", - "accent-emerald-950", - "accent-teal-50", - "accent-teal-100", - "accent-teal-200", - "accent-teal-300", - "accent-teal-400", - "accent-teal-500", - "accent-teal-600", - "accent-teal-700", - "accent-teal-800", - "accent-teal-900", - "accent-teal-950", - "accent-cyan-50", - "accent-cyan-100", - "accent-cyan-200", - "accent-cyan-300", - "accent-cyan-400", - "accent-cyan-500", - "accent-cyan-600", - "accent-cyan-700", - "accent-cyan-800", - "accent-cyan-900", - "accent-cyan-950", - "accent-sky-50", - "accent-sky-100", - "accent-sky-200", - "accent-sky-300", - "accent-sky-400", - "accent-sky-500", - "accent-sky-600", - "accent-sky-700", - "accent-sky-800", - "accent-sky-900", - "accent-sky-950", - "accent-blue-50", - "accent-blue-100", - "accent-blue-200", - "accent-blue-300", - "accent-blue-400", - "accent-blue-500", - "accent-blue-600", - "accent-blue-700", - "accent-blue-800", - "accent-blue-900", - "accent-blue-950", - "accent-indigo-50", - "accent-indigo-100", - "accent-indigo-200", - "accent-indigo-300", - "accent-indigo-400", - "accent-indigo-500", - "accent-indigo-600", - "accent-indigo-700", - "accent-indigo-800", - "accent-indigo-900", - "accent-indigo-950", - "accent-violet-50", - "accent-violet-100", - "accent-violet-200", - "accent-violet-300", - "accent-violet-400", - "accent-violet-500", - "accent-violet-600", - "accent-violet-700", - "accent-violet-800", - "accent-violet-900", - "accent-violet-950", - "accent-purple-50", - "accent-purple-100", - "accent-purple-200", - "accent-purple-300", - "accent-purple-400", - "accent-purple-500", - "accent-purple-600", - "accent-purple-700", - "accent-purple-800", - "accent-purple-900", - "accent-purple-950", - "accent-fuchsia-50", - "accent-fuchsia-100", - "accent-fuchsia-200", - "accent-fuchsia-300", - "accent-fuchsia-400", - "accent-fuchsia-500", - "accent-fuchsia-600", - "accent-fuchsia-700", - "accent-fuchsia-800", - "accent-fuchsia-900", - "accent-fuchsia-950", - "accent-pink-50", - "accent-pink-100", - "accent-pink-200", - "accent-pink-300", - "accent-pink-400", - "accent-pink-500", - "accent-pink-600", - "accent-pink-700", - "accent-pink-800", - "accent-pink-900", - "accent-pink-950", - "accent-rose-50", - "accent-rose-100", - "accent-rose-200", - "accent-rose-300", - "accent-rose-400", - "accent-rose-500", - "accent-rose-600", - "accent-rose-700", - "accent-rose-800", - "accent-rose-900", - "accent-rose-950", - "accent-auto", - "appearance-none", - "appearance-auto", - "cursor-auto", - "cursor-default", - "cursor-pointer", - "cursor-wait", - "cursor-text", - "cursor-move", - "cursor-help", - "cursor-not-allowed", - "cursor-none", - "cursor-context-menu", - "cursor-progress", - "cursor-cell", - "cursor-crosshair", - "cursor-vertical-text", - "cursor-alias", - "cursor-copy", - "cursor-no-drop", - "cursor-grab", - "cursor-grabbing", - "cursor-all-scroll", - "cursor-col-resize", - "cursor-row-resize", - "cursor-n-resize", - "cursor-e-resize", - "cursor-s-resize", - "cursor-w-resize", - "cursor-ne-resize", - "cursor-nw-resize", - "cursor-se-resize", - "cursor-sw-resize", - "cursor-ew-resize", - "cursor-ns-resize", - "cursor-nesw-resize", - "cursor-nwse-resize", - "cursor-zoom-in", - "cursor-zoom-out", - "caret-inherit", - "caret-current", - "caret-transparent", - "caret-black", - "caret-white", - "caret-slate-50", - "caret-slate-100", - "caret-slate-200", - "caret-slate-300", - "caret-slate-400", - "caret-slate-500", - "caret-slate-600", - "caret-slate-700", - "caret-slate-800", - "caret-slate-900", - "caret-slate-950", - "caret-gray-50", - "caret-gray-100", - "caret-gray-200", - "caret-gray-300", - "caret-gray-400", - "caret-gray-500", - "caret-gray-600", - "caret-gray-700", - "caret-gray-800", - "caret-gray-900", - "caret-gray-950", - "caret-zinc-50", - "caret-zinc-100", - "caret-zinc-200", - "caret-zinc-300", - "caret-zinc-400", - "caret-zinc-500", - "caret-zinc-600", - "caret-zinc-700", - "caret-zinc-800", - "caret-zinc-900", - "caret-zinc-950", - "caret-neutral-50", - "caret-neutral-100", - "caret-neutral-200", - "caret-neutral-300", - "caret-neutral-400", - "caret-neutral-500", - "caret-neutral-600", - "caret-neutral-700", - "caret-neutral-800", - "caret-neutral-900", - "caret-neutral-950", - "caret-stone-50", - "caret-stone-100", - "caret-stone-200", - "caret-stone-300", - "caret-stone-400", - "caret-stone-500", - "caret-stone-600", - "caret-stone-700", - "caret-stone-800", - "caret-stone-900", - "caret-stone-950", - "caret-red-50", - "caret-red-100", - "caret-red-200", - "caret-red-300", - "caret-red-400", - "caret-red-500", - "caret-red-600", - "caret-red-700", - "caret-red-800", - "caret-red-900", - "caret-red-950", - "caret-orange-50", - "caret-orange-100", - "caret-orange-200", - "caret-orange-300", - "caret-orange-400", - "caret-orange-500", - "caret-orange-600", - "caret-orange-700", - "caret-orange-800", - "caret-orange-900", - "caret-orange-950", - "caret-amber-50", - "caret-amber-100", - "caret-amber-200", - "caret-amber-300", - "caret-amber-400", - "caret-amber-500", - "caret-amber-600", - "caret-amber-700", - "caret-amber-800", - "caret-amber-900", - "caret-amber-950", - "caret-yellow-50", - "caret-yellow-100", - "caret-yellow-200", - "caret-yellow-300", - "caret-yellow-400", - "caret-yellow-500", - "caret-yellow-600", - "caret-yellow-700", - "caret-yellow-800", - "caret-yellow-900", - "caret-yellow-950", - "caret-lime-50", - "caret-lime-100", - "caret-lime-200", - "caret-lime-300", - "caret-lime-400", - "caret-lime-500", - "caret-lime-600", - "caret-lime-700", - "caret-lime-800", - "caret-lime-900", - "caret-lime-950", - "caret-green-50", - "caret-green-100", - "caret-green-200", - "caret-green-300", - "caret-green-400", - "caret-green-500", - "caret-green-600", - "caret-green-700", - "caret-green-800", - "caret-green-900", - "caret-green-950", - "caret-emerald-50", - "caret-emerald-100", - "caret-emerald-200", - "caret-emerald-300", - "caret-emerald-400", - "caret-emerald-500", - "caret-emerald-600", - "caret-emerald-700", - "caret-emerald-800", - "caret-emerald-900", - "caret-emerald-950", - "caret-teal-50", - "caret-teal-100", - "caret-teal-200", - "caret-teal-300", - "caret-teal-400", - "caret-teal-500", - "caret-teal-600", - "caret-teal-700", - "caret-teal-800", - "caret-teal-900", - "caret-teal-950", - "caret-cyan-50", - "caret-cyan-100", - "caret-cyan-200", - "caret-cyan-300", - "caret-cyan-400", - "caret-cyan-500", - "caret-cyan-600", - "caret-cyan-700", - "caret-cyan-800", - "caret-cyan-900", - "caret-cyan-950", - "caret-sky-50", - "caret-sky-100", - "caret-sky-200", - "caret-sky-300", - "caret-sky-400", - "caret-sky-500", - "caret-sky-600", - "caret-sky-700", - "caret-sky-800", - "caret-sky-900", - "caret-sky-950", - "caret-blue-50", - "caret-blue-100", - "caret-blue-200", - "caret-blue-300", - "caret-blue-400", - "caret-blue-500", - "caret-blue-600", - "caret-blue-700", - "caret-blue-800", - "caret-blue-900", - "caret-blue-950", - "caret-indigo-50", - "caret-indigo-100", - "caret-indigo-200", - "caret-indigo-300", - "caret-indigo-400", - "caret-indigo-500", - "caret-indigo-600", - "caret-indigo-700", - "caret-indigo-800", - "caret-indigo-900", - "caret-indigo-950", - "caret-violet-50", - "caret-violet-100", - "caret-violet-200", - "caret-violet-300", - "caret-violet-400", - "caret-violet-500", - "caret-violet-600", - "caret-violet-700", - "caret-violet-800", - "caret-violet-900", - "caret-violet-950", - "caret-purple-50", - "caret-purple-100", - "caret-purple-200", - "caret-purple-300", - "caret-purple-400", - "caret-purple-500", - "caret-purple-600", - "caret-purple-700", - "caret-purple-800", - "caret-purple-900", - "caret-purple-950", - "caret-fuchsia-50", - "caret-fuchsia-100", - "caret-fuchsia-200", - "caret-fuchsia-300", - "caret-fuchsia-400", - "caret-fuchsia-500", - "caret-fuchsia-600", - "caret-fuchsia-700", - "caret-fuchsia-800", - "caret-fuchsia-900", - "caret-fuchsia-950", - "caret-pink-50", - "caret-pink-100", - "caret-pink-200", - "caret-pink-300", - "caret-pink-400", - "caret-pink-500", - "caret-pink-600", - "caret-pink-700", - "caret-pink-800", - "caret-pink-900", - "caret-pink-950", - "caret-rose-50", - "caret-rose-100", - "caret-rose-200", - "caret-rose-300", - "caret-rose-400", - "caret-rose-500", - "caret-rose-600", - "caret-rose-700", - "caret-rose-800", - "caret-rose-900", - "caret-rose-950", - "pointer-events-none", - "pointer-events-auto", - "resize-none", - "resize-y", - "resize-x", - "resize", - "scroll-auto", - "scroll-smooth", - "scroll-m-0", - "scroll-mx-0", - "scroll-my-0", - "scroll-ms-0", - "scroll-me-0", - "scroll-mt-0", - "scroll-mr-0", - "scroll-mb-0", - "scroll-ml-0", - "scroll-m-px", - "scroll-mx-px", - "scroll-my-px", - "scroll-ms-px", - "scroll-me-px", - "scroll-mt-px", - "scroll-mr-px", - "scroll-mb-px", - "scroll-ml-px", - "scroll-m-0.5", - "scroll-mx-0.5", - "scroll-my-0.5", - "scroll-ms-0.5", - "scroll-me-0.5", - "scroll-mt-0.5", - "scroll-mr-0.5", - "scroll-mb-0.5", - "scroll-ml-0.5", - "scroll-m-1", - "scroll-mx-1", - "scroll-my-1", - "scroll-ms-1", - "scroll-me-1", - "scroll-mt-1", - "scroll-mr-1", - "scroll-mb-1", - "scroll-ml-1", - "scroll-m-1.5", - "scroll-mx-1.5", - "scroll-my-1.5", - "scroll-ms-1.5", - "scroll-me-1.5", - "scroll-mt-1.5", - "scroll-mr-1.5", - "scroll-mb-1.5", - "scroll-ml-1.5", - "scroll-m-2", - "scroll-mx-2", - "scroll-my-2", - "scroll-ms-2", - "scroll-me-2", - "scroll-mt-2", - "scroll-mr-2", - "scroll-mb-2", - "scroll-ml-2", - "scroll-m-2.5", - "scroll-mx-2.5", - "scroll-my-2.5", - "scroll-ms-2.5", - "scroll-me-2.5", - "scroll-mt-2.5", - "scroll-mr-2.5", - "scroll-mb-2.5", - "scroll-ml-2.5", - "scroll-m-3", - "scroll-mx-3", - "scroll-my-3", - "scroll-ms-3", - "scroll-me-3", - "scroll-mt-3", - "scroll-mr-3", - "scroll-mb-3", - "scroll-ml-3", - "scroll-m-3.5", - "scroll-mx-3.5", - "scroll-my-3.5", - "scroll-ms-3.5", - "scroll-me-3.5", - "scroll-mt-3.5", - "scroll-mr-3.5", - "scroll-mb-3.5", - "scroll-ml-3.5", - "scroll-m-4", - "scroll-mx-4", - "scroll-my-4", - "scroll-ms-4", - "scroll-me-4", - "scroll-mt-4", - "scroll-mr-4", - "scroll-mb-4", - "scroll-ml-4", - "scroll-m-5", - "scroll-mx-5", - "scroll-my-5", - "scroll-ms-5", - "scroll-me-5", - "scroll-mt-5", - "scroll-mr-5", - "scroll-mb-5", - "scroll-ml-5", - "scroll-m-6", - "scroll-mx-6", - "scroll-my-6", - "scroll-ms-6", - "scroll-me-6", - "scroll-mt-6", - "scroll-mr-6", - "scroll-mb-6", - "scroll-ml-6", - "scroll-m-7", - "scroll-mx-7", - "scroll-my-7", - "scroll-ms-7", - "scroll-me-7", - "scroll-mt-7", - "scroll-mr-7", - "scroll-mb-7", - "scroll-ml-7", - "scroll-m-8", - "scroll-mx-8", - "scroll-my-8", - "scroll-ms-8", - "scroll-me-8", - "scroll-mt-8", - "scroll-mr-8", - "scroll-mb-8", - "scroll-ml-8", - "scroll-m-9", - "scroll-mx-9", - "scroll-my-9", - "scroll-ms-9", - "scroll-me-9", - "scroll-mt-9", - "scroll-mr-9", - "scroll-mb-9", - "scroll-ml-9", - "scroll-m-10", - "scroll-mx-10", - "scroll-my-10", - "scroll-ms-10", - "scroll-me-10", - "scroll-mt-10", - "scroll-mr-10", - "scroll-mb-10", - "scroll-ml-10", - "scroll-m-11", - "scroll-mx-11", - "scroll-my-11", - "scroll-ms-11", - "scroll-me-11", - "scroll-mt-11", - "scroll-mr-11", - "scroll-mb-11", - "scroll-ml-11", - "scroll-m-12", - "scroll-mx-12", - "scroll-my-12", - "scroll-ms-12", - "scroll-me-12", - "scroll-mt-12", - "scroll-mr-12", - "scroll-mb-12", - "scroll-ml-12", - "scroll-m-14", - "scroll-mx-14", - "scroll-my-14", - "scroll-ms-14", - "scroll-me-14", - "scroll-mt-14", - "scroll-mr-14", - "scroll-mb-14", - "scroll-ml-14", - "scroll-m-16", - "scroll-mx-16", - "scroll-my-16", - "scroll-ms-16", - "scroll-me-16", - "scroll-mt-16", - "scroll-mr-16", - "scroll-mb-16", - "scroll-ml-16", - "scroll-m-20", - "scroll-mx-20", - "scroll-my-20", - "scroll-ms-20", - "scroll-me-20", - "scroll-mt-20", - "scroll-mr-20", - "scroll-mb-20", - "scroll-ml-20", - "scroll-m-24", - "scroll-mx-24", - "scroll-my-24", - "scroll-ms-24", - "scroll-me-24", - "scroll-mt-24", - "scroll-mr-24", - "scroll-mb-24", - "scroll-ml-24", - "scroll-m-28", - "scroll-mx-28", - "scroll-my-28", - "scroll-ms-28", - "scroll-me-28", - "scroll-mt-28", - "scroll-mr-28", - "scroll-mb-28", - "scroll-ml-28", - "scroll-m-32", - "scroll-mx-32", - "scroll-my-32", - "scroll-ms-32", - "scroll-me-32", - "scroll-mt-32", - "scroll-mr-32", - "scroll-mb-32", - "scroll-ml-32", - "scroll-m-36", - "scroll-mx-36", - "scroll-my-36", - "scroll-ms-36", - "scroll-me-36", - "scroll-mt-36", - "scroll-mr-36", - "scroll-mb-36", - "scroll-ml-36", - "scroll-m-40", - "scroll-mx-40", - "scroll-my-40", - "scroll-ms-40", - "scroll-me-40", - "scroll-mt-40", - "scroll-mr-40", - "scroll-mb-40", - "scroll-ml-40", - "scroll-m-44", - "scroll-mx-44", - "scroll-my-44", - "scroll-ms-44", - "scroll-me-44", - "scroll-mt-44", - "scroll-mr-44", - "scroll-mb-44", - "scroll-ml-44", - "scroll-m-48", - "scroll-mx-48", - "scroll-my-48", - "scroll-ms-48", - "scroll-me-48", - "scroll-mt-48", - "scroll-mr-48", - "scroll-mb-48", - "scroll-ml-48", - "scroll-m-52", - "scroll-mx-52", - "scroll-my-52", - "scroll-ms-52", - "scroll-me-52", - "scroll-mt-52", - "scroll-mr-52", - "scroll-mb-52", - "scroll-ml-52", - "scroll-m-56", - "scroll-mx-56", - "scroll-my-56", - "scroll-ms-56", - "scroll-me-56", - "scroll-mt-56", - "scroll-mr-56", - "scroll-mb-56", - "scroll-ml-56", - "scroll-m-60", - "scroll-mx-60", - "scroll-my-60", - "scroll-ms-60", - "scroll-me-60", - "scroll-mt-60", - "scroll-mr-60", - "scroll-mb-60", - "scroll-ml-60", - "scroll-m-64", - "scroll-mx-64", - "scroll-my-64", - "scroll-ms-64", - "scroll-me-64", - "scroll-mt-64", - "scroll-mr-64", - "scroll-mb-64", - "scroll-ml-64", - "scroll-m-72", - "scroll-mx-72", - "scroll-my-72", - "scroll-ms-72", - "scroll-me-72", - "scroll-mt-72", - "scroll-mr-72", - "scroll-mb-72", - "scroll-ml-72", - "scroll-m-80", - "scroll-mx-80", - "scroll-my-80", - "scroll-ms-80", - "scroll-me-80", - "scroll-mt-80", - "scroll-mr-80", - "scroll-mb-80", - "scroll-ml-80", - "scroll-m-96", - "scroll-mx-96", - "scroll-my-96", - "scroll-ms-96", - "scroll-me-96", - "scroll-mt-96", - "scroll-mr-96", - "scroll-mb-96", - "scroll-ml-96", - "scroll-p-0", - "scroll-px-0", - "scroll-py-0", - "scroll-ps-0", - "scroll-pe-0", - "scroll-pt-0", - "scroll-pr-0", - "scroll-pb-0", - "scroll-pl-0", - "scroll-p-px", - "scroll-px-px", - "scroll-py-px", - "scroll-ps-px", - "scroll-pe-px", - "scroll-pt-px", - "scroll-pr-px", - "scroll-pb-px", - "scroll-pl-px", - "scroll-p-0.5", - "scroll-px-0.5", - "scroll-py-0.5", - "scroll-ps-0.5", - "scroll-pe-0.5", - "scroll-pt-0.5", - "scroll-pr-0.5", - "scroll-pb-0.5", - "scroll-pl-0.5", - "scroll-p-1", - "scroll-px-1", - "scroll-py-1", - "scroll-ps-1", - "scroll-pe-1", - "scroll-pt-1", - "scroll-pr-1", - "scroll-pb-1", - "scroll-pl-1", - "scroll-p-1.5", - "scroll-px-1.5", - "scroll-py-1.5", - "scroll-ps-1.5", - "scroll-pe-1.5", - "scroll-pt-1.5", - "scroll-pr-1.5", - "scroll-pb-1.5", - "scroll-pl-1.5", - "scroll-p-2", - "scroll-px-2", - "scroll-py-2", - "scroll-ps-2", - "scroll-pe-2", - "scroll-pt-2", - "scroll-pr-2", - "scroll-pb-2", - "scroll-pl-2", - "scroll-p-2.5", - "scroll-px-2.5", - "scroll-py-2.5", - "scroll-ps-2.5", - "scroll-pe-2.5", - "scroll-pt-2.5", - "scroll-pr-2.5", - "scroll-pb-2.5", - "scroll-pl-2.5", - "scroll-p-3", - "scroll-px-3", - "scroll-py-3", - "scroll-ps-3", - "scroll-pe-3", - "scroll-pt-3", - "scroll-pr-3", - "scroll-pb-3", - "scroll-pl-3", - "scroll-p-3.5", - "scroll-px-3.5", - "scroll-py-3.5", - "scroll-ps-3.5", - "scroll-pe-3.5", - "scroll-pt-3.5", - "scroll-pr-3.5", - "scroll-pb-3.5", - "scroll-pl-3.5", - "scroll-p-4", - "scroll-px-4", - "scroll-py-4", - "scroll-ps-4", - "scroll-pe-4", - "scroll-pt-4", - "scroll-pr-4", - "scroll-pb-4", - "scroll-pl-4", - "scroll-p-5", - "scroll-px-5", - "scroll-py-5", - "scroll-ps-5", - "scroll-pe-5", - "scroll-pt-5", - "scroll-pr-5", - "scroll-pb-5", - "scroll-pl-5", - "scroll-p-6", - "scroll-px-6", - "scroll-py-6", - "scroll-ps-6", - "scroll-pe-6", - "scroll-pt-6", - "scroll-pr-6", - "scroll-pb-6", - "scroll-pl-6", - "scroll-p-7", - "scroll-px-7", - "scroll-py-7", - "scroll-ps-7", - "scroll-pe-7", - "scroll-pt-7", - "scroll-pr-7", - "scroll-pb-7", - "scroll-pl-7", - "scroll-p-8", - "scroll-px-8", - "scroll-py-8", - "scroll-ps-8", - "scroll-pe-8", - "scroll-pt-8", - "scroll-pr-8", - "scroll-pb-8", - "scroll-pl-8", - "scroll-p-9", - "scroll-px-9", - "scroll-py-9", - "scroll-ps-9", - "scroll-pe-9", - "scroll-pt-9", - "scroll-pr-9", - "scroll-pb-9", - "scroll-pl-9", - "scroll-p-10", - "scroll-px-10", - "scroll-py-10", - "scroll-ps-10", - "scroll-pe-10", - "scroll-pt-10", - "scroll-pr-10", - "scroll-pb-10", - "scroll-pl-10", - "scroll-p-11", - "scroll-px-11", - "scroll-py-11", - "scroll-ps-11", - "scroll-pe-11", - "scroll-pt-11", - "scroll-pr-11", - "scroll-pb-11", - "scroll-pl-11", - "scroll-p-12", - "scroll-px-12", - "scroll-py-12", - "scroll-ps-12", - "scroll-pe-12", - "scroll-pt-12", - "scroll-pr-12", - "scroll-pb-12", - "scroll-pl-12", - "scroll-p-14", - "scroll-px-14", - "scroll-py-14", - "scroll-ps-14", - "scroll-pe-14", - "scroll-pt-14", - "scroll-pr-14", - "scroll-pb-14", - "scroll-pl-14", - "scroll-p-16", - "scroll-px-16", - "scroll-py-16", - "scroll-ps-16", - "scroll-pe-16", - "scroll-pt-16", - "scroll-pr-16", - "scroll-pb-16", - "scroll-pl-16", - "scroll-p-20", - "scroll-px-20", - "scroll-py-20", - "scroll-ps-20", - "scroll-pe-20", - "scroll-pt-20", - "scroll-pr-20", - "scroll-pb-20", - "scroll-pl-20", - "scroll-p-24", - "scroll-px-24", - "scroll-py-24", - "scroll-ps-24", - "scroll-pe-24", - "scroll-pt-24", - "scroll-pr-24", - "scroll-pb-24", - "scroll-pl-24", - "scroll-p-28", - "scroll-px-28", - "scroll-py-28", - "scroll-ps-28", - "scroll-pe-28", - "scroll-pt-28", - "scroll-pr-28", - "scroll-pb-28", - "scroll-pl-28", - "scroll-p-32", - "scroll-px-32", - "scroll-py-32", - "scroll-ps-32", - "scroll-pe-32", - "scroll-pt-32", - "scroll-pr-32", - "scroll-pb-32", - "scroll-pl-32", - "scroll-p-36", - "scroll-px-36", - "scroll-py-36", - "scroll-ps-36", - "scroll-pe-36", - "scroll-pt-36", - "scroll-pr-36", - "scroll-pb-36", - "scroll-pl-36", - "scroll-p-40", - "scroll-px-40", - "scroll-py-40", - "scroll-ps-40", - "scroll-pe-40", - "scroll-pt-40", - "scroll-pr-40", - "scroll-pb-40", - "scroll-pl-40", - "scroll-p-44", - "scroll-px-44", - "scroll-py-44", - "scroll-ps-44", - "scroll-pe-44", - "scroll-pt-44", - "scroll-pr-44", - "scroll-pb-44", - "scroll-pl-44", - "scroll-p-48", - "scroll-px-48", - "scroll-py-48", - "scroll-ps-48", - "scroll-pe-48", - "scroll-pt-48", - "scroll-pr-48", - "scroll-pb-48", - "scroll-pl-48", - "scroll-p-52", - "scroll-px-52", - "scroll-py-52", - "scroll-ps-52", - "scroll-pe-52", - "scroll-pt-52", - "scroll-pr-52", - "scroll-pb-52", - "scroll-pl-52", - "scroll-p-56", - "scroll-px-56", - "scroll-py-56", - "scroll-ps-56", - "scroll-pe-56", - "scroll-pt-56", - "scroll-pr-56", - "scroll-pb-56", - "scroll-pl-56", - "scroll-p-60", - "scroll-px-60", - "scroll-py-60", - "scroll-ps-60", - "scroll-pe-60", - "scroll-pt-60", - "scroll-pr-60", - "scroll-pb-60", - "scroll-pl-60", - "scroll-p-64", - "scroll-px-64", - "scroll-py-64", - "scroll-ps-64", - "scroll-pe-64", - "scroll-pt-64", - "scroll-pr-64", - "scroll-pb-64", - "scroll-pl-64", - "scroll-p-72", - "scroll-px-72", - "scroll-py-72", - "scroll-ps-72", - "scroll-pe-72", - "scroll-pt-72", - "scroll-pr-72", - "scroll-pb-72", - "scroll-pl-72", - "scroll-p-80", - "scroll-px-80", - "scroll-py-80", - "scroll-ps-80", - "scroll-pe-80", - "scroll-pt-80", - "scroll-pr-80", - "scroll-pb-80", - "scroll-pl-80", - "scroll-p-96", - "scroll-px-96", - "scroll-py-96", - "scroll-ps-96", - "scroll-pe-96", - "scroll-pt-96", - "scroll-pr-96", - "scroll-pb-96", - "scroll-pl-96", - "snap-start", - "snap-end", - "snap-center", - "snap-align-none", - "snap-normal", - "snap-always", - "snap-none", - "snap-x", - "snap-y", - "snap-both", - "snap-mandatory", - "snap-proximity", - "touch-auto", - "touch-none", - "touch-pan-x", - "touch-pan-left", - "touch-pan-right", - "touch-pan-y", - "touch-pan-up", - "touch-pan-down", - "touch-pinch-zoom", - "touch-manipulation", - "select-none", - "select-text", - "select-all", - "select-auto", - "will-change-auto", - "will-change-scroll", - "will-change-contents", - "will-change-transform", - "fill-none", - "fill-inherit", - "fill-current", - "fill-transparent", - "fill-black", - "fill-white", - "fill-slate-50", - "fill-slate-100", - "fill-slate-200", - "fill-slate-300", - "fill-slate-400", - "fill-slate-500", - "fill-slate-600", - "fill-slate-700", - "fill-slate-800", - "fill-slate-900", - "fill-slate-950", - "fill-gray-50", - "fill-gray-100", - "fill-gray-200", - "fill-gray-300", - "fill-gray-400", - "fill-gray-500", - "fill-gray-600", - "fill-gray-700", - "fill-gray-800", - "fill-gray-900", - "fill-gray-950", - "fill-zinc-50", - "fill-zinc-100", - "fill-zinc-200", - "fill-zinc-300", - "fill-zinc-400", - "fill-zinc-500", - "fill-zinc-600", - "fill-zinc-700", - "fill-zinc-800", - "fill-zinc-900", - "fill-zinc-950", - "fill-neutral-50", - "fill-neutral-100", - "fill-neutral-200", - "fill-neutral-300", - "fill-neutral-400", - "fill-neutral-500", - "fill-neutral-600", - "fill-neutral-700", - "fill-neutral-800", - "fill-neutral-900", - "fill-neutral-950", - "fill-stone-50", - "fill-stone-100", - "fill-stone-200", - "fill-stone-300", - "fill-stone-400", - "fill-stone-500", - "fill-stone-600", - "fill-stone-700", - "fill-stone-800", - "fill-stone-900", - "fill-stone-950", - "fill-red-50", - "fill-red-100", - "fill-red-200", - "fill-red-300", - "fill-red-400", - "fill-red-500", - "fill-red-600", - "fill-red-700", - "fill-red-800", - "fill-red-900", - "fill-red-950", - "fill-orange-50", - "fill-orange-100", - "fill-orange-200", - "fill-orange-300", - "fill-orange-400", - "fill-orange-500", - "fill-orange-600", - "fill-orange-700", - "fill-orange-800", - "fill-orange-900", - "fill-orange-950", - "fill-amber-50", - "fill-amber-100", - "fill-amber-200", - "fill-amber-300", - "fill-amber-400", - "fill-amber-500", - "fill-amber-600", - "fill-amber-700", - "fill-amber-800", - "fill-amber-900", - "fill-amber-950", - "fill-yellow-50", - "fill-yellow-100", - "fill-yellow-200", - "fill-yellow-300", - "fill-yellow-400", - "fill-yellow-500", - "fill-yellow-600", - "fill-yellow-700", - "fill-yellow-800", - "fill-yellow-900", - "fill-yellow-950", - "fill-lime-50", - "fill-lime-100", - "fill-lime-200", - "fill-lime-300", - "fill-lime-400", - "fill-lime-500", - "fill-lime-600", - "fill-lime-700", - "fill-lime-800", - "fill-lime-900", - "fill-lime-950", - "fill-green-50", - "fill-green-100", - "fill-green-200", - "fill-green-300", - "fill-green-400", - "fill-green-500", - "fill-green-600", - "fill-green-700", - "fill-green-800", - "fill-green-900", - "fill-green-950", - "fill-emerald-50", - "fill-emerald-100", - "fill-emerald-200", - "fill-emerald-300", - "fill-emerald-400", - "fill-emerald-500", - "fill-emerald-600", - "fill-emerald-700", - "fill-emerald-800", - "fill-emerald-900", - "fill-emerald-950", - "fill-teal-50", - "fill-teal-100", - "fill-teal-200", - "fill-teal-300", - "fill-teal-400", - "fill-teal-500", - "fill-teal-600", - "fill-teal-700", - "fill-teal-800", - "fill-teal-900", - "fill-teal-950", - "fill-cyan-50", - "fill-cyan-100", - "fill-cyan-200", - "fill-cyan-300", - "fill-cyan-400", - "fill-cyan-500", - "fill-cyan-600", - "fill-cyan-700", - "fill-cyan-800", - "fill-cyan-900", - "fill-cyan-950", - "fill-sky-50", - "fill-sky-100", - "fill-sky-200", - "fill-sky-300", - "fill-sky-400", - "fill-sky-500", - "fill-sky-600", - "fill-sky-700", - "fill-sky-800", - "fill-sky-900", - "fill-sky-950", - "fill-blue-50", - "fill-blue-100", - "fill-blue-200", - "fill-blue-300", - "fill-blue-400", - "fill-blue-500", - "fill-blue-600", - "fill-blue-700", - "fill-blue-800", - "fill-blue-900", - "fill-blue-950", - "fill-indigo-50", - "fill-indigo-100", - "fill-indigo-200", - "fill-indigo-300", - "fill-indigo-400", - "fill-indigo-500", - "fill-indigo-600", - "fill-indigo-700", - "fill-indigo-800", - "fill-indigo-900", - "fill-indigo-950", - "fill-violet-50", - "fill-violet-100", - "fill-violet-200", - "fill-violet-300", - "fill-violet-400", - "fill-violet-500", - "fill-violet-600", - "fill-violet-700", - "fill-violet-800", - "fill-violet-900", - "fill-violet-950", - "fill-purple-50", - "fill-purple-100", - "fill-purple-200", - "fill-purple-300", - "fill-purple-400", - "fill-purple-500", - "fill-purple-600", - "fill-purple-700", - "fill-purple-800", - "fill-purple-900", - "fill-purple-950", - "fill-fuchsia-50", - "fill-fuchsia-100", - "fill-fuchsia-200", - "fill-fuchsia-300", - "fill-fuchsia-400", - "fill-fuchsia-500", - "fill-fuchsia-600", - "fill-fuchsia-700", - "fill-fuchsia-800", - "fill-fuchsia-900", - "fill-fuchsia-950", - "fill-pink-50", - "fill-pink-100", - "fill-pink-200", - "fill-pink-300", - "fill-pink-400", - "fill-pink-500", - "fill-pink-600", - "fill-pink-700", - "fill-pink-800", - "fill-pink-900", - "fill-pink-950", - "fill-rose-50", - "fill-rose-100", - "fill-rose-200", - "fill-rose-300", - "fill-rose-400", - "fill-rose-500", - "fill-rose-600", - "fill-rose-700", - "fill-rose-800", - "fill-rose-900", - "fill-rose-950", - "stroke-none", - "stroke-inherit", - "stroke-current", - "stroke-transparent", - "stroke-black", - "stroke-white", - "stroke-slate-50", - "stroke-slate-100", - "stroke-slate-200", - "stroke-slate-300", - "stroke-slate-400", - "stroke-slate-500", - "stroke-slate-600", - "stroke-slate-700", - "stroke-slate-800", - "stroke-slate-900", - "stroke-slate-950", - "stroke-gray-50", - "stroke-gray-100", - "stroke-gray-200", - "stroke-gray-300", - "stroke-gray-400", - "stroke-gray-500", - "stroke-gray-600", - "stroke-gray-700", - "stroke-gray-800", - "stroke-gray-900", - "stroke-gray-950", - "stroke-zinc-50", - "stroke-zinc-100", - "stroke-zinc-200", - "stroke-zinc-300", - "stroke-zinc-400", - "stroke-zinc-500", - "stroke-zinc-600", - "stroke-zinc-700", - "stroke-zinc-800", - "stroke-zinc-900", - "stroke-zinc-950", - "stroke-neutral-50", - "stroke-neutral-100", - "stroke-neutral-200", - "stroke-neutral-300", - "stroke-neutral-400", - "stroke-neutral-500", - "stroke-neutral-600", - "stroke-neutral-700", - "stroke-neutral-800", - "stroke-neutral-900", - "stroke-neutral-950", - "stroke-stone-50", - "stroke-stone-100", - "stroke-stone-200", - "stroke-stone-300", - "stroke-stone-400", - "stroke-stone-500", - "stroke-stone-600", - "stroke-stone-700", - "stroke-stone-800", - "stroke-stone-900", - "stroke-stone-950", - "stroke-red-50", - "stroke-red-100", - "stroke-red-200", - "stroke-red-300", - "stroke-red-400", - "stroke-red-500", - "stroke-red-600", - "stroke-red-700", - "stroke-red-800", - "stroke-red-900", - "stroke-red-950", - "stroke-orange-50", - "stroke-orange-100", - "stroke-orange-200", - "stroke-orange-300", - "stroke-orange-400", - "stroke-orange-500", - "stroke-orange-600", - "stroke-orange-700", - "stroke-orange-800", - "stroke-orange-900", - "stroke-orange-950", - "stroke-amber-50", - "stroke-amber-100", - "stroke-amber-200", - "stroke-amber-300", - "stroke-amber-400", - "stroke-amber-500", - "stroke-amber-600", - "stroke-amber-700", - "stroke-amber-800", - "stroke-amber-900", - "stroke-amber-950", - "stroke-yellow-50", - "stroke-yellow-100", - "stroke-yellow-200", - "stroke-yellow-300", - "stroke-yellow-400", - "stroke-yellow-500", - "stroke-yellow-600", - "stroke-yellow-700", - "stroke-yellow-800", - "stroke-yellow-900", - "stroke-yellow-950", - "stroke-lime-50", - "stroke-lime-100", - "stroke-lime-200", - "stroke-lime-300", - "stroke-lime-400", - "stroke-lime-500", - "stroke-lime-600", - "stroke-lime-700", - "stroke-lime-800", - "stroke-lime-900", - "stroke-lime-950", - "stroke-green-50", - "stroke-green-100", - "stroke-green-200", - "stroke-green-300", - "stroke-green-400", - "stroke-green-500", - "stroke-green-600", - "stroke-green-700", - "stroke-green-800", - "stroke-green-900", - "stroke-green-950", - "stroke-emerald-50", - "stroke-emerald-100", - "stroke-emerald-200", - "stroke-emerald-300", - "stroke-emerald-400", - "stroke-emerald-500", - "stroke-emerald-600", - "stroke-emerald-700", - "stroke-emerald-800", - "stroke-emerald-900", - "stroke-emerald-950", - "stroke-teal-50", - "stroke-teal-100", - "stroke-teal-200", - "stroke-teal-300", - "stroke-teal-400", - "stroke-teal-500", - "stroke-teal-600", - "stroke-teal-700", - "stroke-teal-800", - "stroke-teal-900", - "stroke-teal-950", - "stroke-cyan-50", - "stroke-cyan-100", - "stroke-cyan-200", - "stroke-cyan-300", - "stroke-cyan-400", - "stroke-cyan-500", - "stroke-cyan-600", - "stroke-cyan-700", - "stroke-cyan-800", - "stroke-cyan-900", - "stroke-cyan-950", - "stroke-sky-50", - "stroke-sky-100", - "stroke-sky-200", - "stroke-sky-300", - "stroke-sky-400", - "stroke-sky-500", - "stroke-sky-600", - "stroke-sky-700", - "stroke-sky-800", - "stroke-sky-900", - "stroke-sky-950", - "stroke-blue-50", - "stroke-blue-100", - "stroke-blue-200", - "stroke-blue-300", - "stroke-blue-400", - "stroke-blue-500", - "stroke-blue-600", - "stroke-blue-700", - "stroke-blue-800", - "stroke-blue-900", - "stroke-blue-950", - "stroke-indigo-50", - "stroke-indigo-100", - "stroke-indigo-200", - "stroke-indigo-300", - "stroke-indigo-400", - "stroke-indigo-500", - "stroke-indigo-600", - "stroke-indigo-700", - "stroke-indigo-800", - "stroke-indigo-900", - "stroke-indigo-950", - "stroke-violet-50", - "stroke-violet-100", - "stroke-violet-200", - "stroke-violet-300", - "stroke-violet-400", - "stroke-violet-500", - "stroke-violet-600", - "stroke-violet-700", - "stroke-violet-800", - "stroke-violet-900", - "stroke-violet-950", - "stroke-purple-50", - "stroke-purple-100", - "stroke-purple-200", - "stroke-purple-300", - "stroke-purple-400", - "stroke-purple-500", - "stroke-purple-600", - "stroke-purple-700", - "stroke-purple-800", - "stroke-purple-900", - "stroke-purple-950", - "stroke-fuchsia-50", - "stroke-fuchsia-100", - "stroke-fuchsia-200", - "stroke-fuchsia-300", - "stroke-fuchsia-400", - "stroke-fuchsia-500", - "stroke-fuchsia-600", - "stroke-fuchsia-700", - "stroke-fuchsia-800", - "stroke-fuchsia-900", - "stroke-fuchsia-950", - "stroke-pink-50", - "stroke-pink-100", - "stroke-pink-200", - "stroke-pink-300", - "stroke-pink-400", - "stroke-pink-500", - "stroke-pink-600", - "stroke-pink-700", - "stroke-pink-800", - "stroke-pink-900", - "stroke-pink-950", - "stroke-rose-50", - "stroke-rose-100", - "stroke-rose-200", - "stroke-rose-300", - "stroke-rose-400", - "stroke-rose-500", - "stroke-rose-600", - "stroke-rose-700", - "stroke-rose-800", - "stroke-rose-900", - "stroke-rose-950", - "stroke-0", - "stroke-1", - "stroke-2", - "sr-only", - "not-sr-only", - "forced-color-adjust-auto", - "forced-color-adjust-none", - "transform", - "transform-none", - "transform-gpu", - "transform-cpu", - "group", - "peer" + "aspect-auto", + "aspect-square", + "aspect-video", + "container", + "columns-1", + "columns-2", + "columns-3", + "columns-4", + "columns-5", + "columns-6", + "columns-7", + "columns-8", + "columns-9", + "columns-10", + "columns-11", + "columns-12", + "columns-auto", + "columns-3xs", + "columns-2xs", + "columns-xs", + "columns-sm", + "columns-md", + "columns-lg", + "columns-xl", + "columns-2xl", + "columns-3xl", + "columns-4xl", + "columns-5xl", + "columns-6xl", + "columns-7xl", + "break-after-auto", + "break-after-avoid", + "break-after-all", + "break-after-avoid-page", + "break-after-page", + "break-after-left", + "break-after-right", + "break-after-column", + "break-before-auto", + "break-before-avoid", + "break-before-all", + "break-before-avoid-page", + "break-before-page", + "break-before-left", + "break-before-right", + "break-before-column", + "break-inside-auto", + "break-inside-avoid", + "break-inside-avoid-page", + "break-inside-avoid-column", + "box-decoration-clone", + "box-decoration-slice", + "box-border", + "box-content", + "block", + "inline-block", + "inline", + "flex", + "inline-flex", + "table", + "inline-table", + "table-caption", + "table-cell", + "table-column", + "table-column-group", + "table-footer-group", + "table-header-group", + "table-row-group", + "table-row", + "flow-root", + "grid", + "inline-grid", + "contents", + "list-item", + // "hidden", + "float-start", + "float-end", + "float-right", + "float-left", + "float-none", + "clear-start", + "clear-end", + "clear-left", + "clear-right", + "clear-both", + "clear-none", + "isolate", + "isolation-auto", + "object-contain", + "object-cover", + "object-fill", + "object-none", + "object-scale-down", + "object-bottom", + "object-center", + "object-left", + "object-left-bottom", + "object-left-top", + "object-right", + "object-right-bottom", + "object-right-top", + "object-top", + "overflow-auto", + "overflow-hidden", + "overflow-clip", + "overflow-visible", + "overflow-scroll", + "overflow-x-auto", + "overflow-y-auto", + "overflow-x-hidden", + "overflow-y-hidden", + "overflow-x-clip", + "overflow-y-clip", + "overflow-x-visible", + "overflow-y-visible", + "overflow-x-scroll", + "overflow-y-scroll", + "overscroll-auto", + "overscroll-contain", + "overscroll-none", + "overscroll-y-auto", + "overscroll-y-contain", + "overscroll-y-none", + "overscroll-x-auto", + "overscroll-x-contain", + "overscroll-x-none", + "static", + "fixed", + "absolute", + "relative", + "sticky", + "inset-0", + "inset-x-0", + "inset-y-0", + "start-0", + "end-0", + "top-0", + "right-0", + "bottom-0", + "left-0", + "inset-px", + "inset-x-px", + "inset-y-px", + "start-px", + "end-px", + "top-px", + "right-px", + "bottom-px", + "left-px", + "inset-0.5", + "inset-x-0.5", + "inset-y-0.5", + "start-0.5", + "end-0.5", + "top-0.5", + "right-0.5", + "bottom-0.5", + "left-0.5", + "inset-1", + "inset-x-1", + "inset-y-1", + "start-1", + "end-1", + "top-1", + "right-1", + "bottom-1", + "left-1", + "inset-1.5", + "inset-x-1.5", + "inset-y-1.5", + "start-1.5", + "end-1.5", + "top-1.5", + "right-1.5", + "bottom-1.5", + "left-1.5", + "inset-2", + "inset-x-2", + "inset-y-2", + "start-2", + "end-2", + "top-2", + "right-2", + "bottom-2", + "left-2", + "inset-2.5", + "inset-x-2.5", + "inset-y-2.5", + "start-2.5", + "end-2.5", + "top-2.5", + "right-2.5", + "bottom-2.5", + "left-2.5", + "inset-3", + "inset-x-3", + "inset-y-3", + "start-3", + "end-3", + "top-3", + "right-3", + "bottom-3", + "left-3", + "inset-3.5", + "inset-x-3.5", + "inset-y-3.5", + "start-3.5", + "end-3.5", + "top-3.5", + "right-3.5", + "bottom-3.5", + "left-3.5", + "inset-4", + "inset-x-4", + "inset-y-4", + "start-4", + "end-4", + "top-4", + "right-4", + "bottom-4", + "left-4", + "inset-5", + "inset-x-5", + "inset-y-5", + "start-5", + "end-5", + "top-5", + "right-5", + "bottom-5", + "left-5", + "inset-6", + "inset-x-6", + "inset-y-6", + "start-6", + "end-6", + "top-6", + "right-6", + "bottom-6", + "left-6", + "inset-7", + "inset-x-7", + "inset-y-7", + "start-7", + "end-7", + "top-7", + "right-7", + "bottom-7", + "left-7", + "inset-8", + "inset-x-8", + "inset-y-8", + "start-8", + "end-8", + "top-8", + "right-8", + "bottom-8", + "left-8", + "inset-9", + "inset-x-9", + "inset-y-9", + "start-9", + "end-9", + "top-9", + "right-9", + "bottom-9", + "left-9", + "inset-10", + "inset-x-10", + "inset-y-10", + "start-10", + "end-10", + "top-10", + "right-10", + "bottom-10", + "left-10", + "inset-11", + "inset-x-11", + "inset-y-11", + "start-11", + "end-11", + "top-11", + "right-11", + "bottom-11", + "left-11", + "inset-12", + "inset-x-12", + "inset-y-12", + "start-12", + "end-12", + "top-12", + "right-12", + "bottom-12", + "left-12", + "inset-14", + "inset-x-14", + "inset-y-14", + "start-14", + "end-14", + "top-14", + "right-14", + "bottom-14", + "left-14", + "inset-16", + "inset-x-16", + "inset-y-16", + "start-16", + "end-16", + "top-16", + "right-16", + "bottom-16", + "left-16", + "inset-20", + "inset-x-20", + "inset-y-20", + "start-20", + "end-20", + "top-20", + "right-20", + "bottom-20", + "left-20", + "inset-24", + "inset-x-24", + "inset-y-24", + "start-24", + "end-24", + "top-24", + "right-24", + "bottom-24", + "left-24", + "inset-28", + "inset-x-28", + "inset-y-28", + "start-28", + "end-28", + "top-28", + "right-28", + "bottom-28", + "left-28", + "inset-32", + "inset-x-32", + "inset-y-32", + "start-32", + "end-32", + "top-32", + "right-32", + "bottom-32", + "left-32", + "inset-36", + "inset-x-36", + "inset-y-36", + "start-36", + "end-36", + "top-36", + "right-36", + "bottom-36", + "left-36", + "inset-40", + "inset-x-40", + "inset-y-40", + "start-40", + "end-40", + "top-40", + "right-40", + "bottom-40", + "left-40", + "inset-44", + "inset-x-44", + "inset-y-44", + "start-44", + "end-44", + "top-44", + "right-44", + "bottom-44", + "left-44", + "inset-48", + "inset-x-48", + "inset-y-48", + "start-48", + "end-48", + "top-48", + "right-48", + "bottom-48", + "left-48", + "inset-52", + "inset-x-52", + "inset-y-52", + "start-52", + "end-52", + "top-52", + "right-52", + "bottom-52", + "left-52", + "inset-56", + "inset-x-56", + "inset-y-56", + "start-56", + "end-56", + "top-56", + "right-56", + "bottom-56", + "left-56", + "inset-60", + "inset-x-60", + "inset-y-60", + "start-60", + "end-60", + "top-60", + "right-60", + "bottom-60", + "left-60", + "inset-64", + "inset-x-64", + "inset-y-64", + "start-64", + "end-64", + "top-64", + "right-64", + "bottom-64", + "left-64", + "inset-72", + "inset-x-72", + "inset-y-72", + "start-72", + "end-72", + "top-72", + "right-72", + "bottom-72", + "left-72", + "inset-80", + "inset-x-80", + "inset-y-80", + "start-80", + "end-80", + "top-80", + "right-80", + "bottom-80", + "left-80", + "inset-96", + "inset-x-96", + "inset-y-96", + "start-96", + "end-96", + "top-96", + "right-96", + "bottom-96", + "left-96", + "inset-auto", + "inset-1/2", + "inset-1/3", + "inset-2/3", + "inset-1/4", + "inset-2/4", + "inset-3/4", + "inset-full", + "inset-x-auto", + "inset-x-1/2", + "inset-x-1/3", + "inset-x-2/3", + "inset-x-1/4", + "inset-x-2/4", + "inset-x-3/4", + "inset-x-full", + "inset-y-auto", + "inset-y-1/2", + "inset-y-1/3", + "inset-y-2/3", + "inset-y-1/4", + "inset-y-2/4", + "inset-y-3/4", + "inset-y-full", + "start-auto", + "start-1/2", + "start-1/3", + "start-2/3", + "start-1/4", + "start-2/4", + "start-3/4", + "start-full", + "end-auto", + "end-1/2", + "end-1/3", + "end-2/3", + "end-1/4", + "end-2/4", + "end-3/4", + "end-full", + "top-auto", + "top-1/2", + "top-1/3", + "top-2/3", + "top-1/4", + "top-2/4", + "top-3/4", + "top-full", + "right-auto", + "right-1/2", + "right-1/3", + "right-2/3", + "right-1/4", + "right-2/4", + "right-3/4", + "right-full", + "bottom-auto", + "bottom-1/2", + "bottom-1/3", + "bottom-2/3", + "bottom-1/4", + "bottom-2/4", + "bottom-3/4", + "bottom-full", + "left-auto", + "left-1/2", + "left-1/3", + "left-2/3", + "left-1/4", + "left-2/4", + "left-3/4", + "left-full", + "visible", + "invisible", + "collapse", + "z-0", + "z-10", + "z-20", + "z-30", + "z-40", + "z-50", + "z-auto", + "basis-0", + "basis-1", + "basis-2", + "basis-3", + "basis-4", + "basis-5", + "basis-6", + "basis-7", + "basis-8", + "basis-9", + "basis-10", + "basis-11", + "basis-12", + "basis-14", + "basis-16", + "basis-20", + "basis-24", + "basis-28", + "basis-32", + "basis-36", + "basis-40", + "basis-44", + "basis-48", + "basis-52", + "basis-56", + "basis-60", + "basis-64", + "basis-72", + "basis-80", + "basis-96", + "basis-auto", + "basis-px", + "basis-0.5", + "basis-1.5", + "basis-2.5", + "basis-3.5", + "basis-1/2", + "basis-1/3", + "basis-2/3", + "basis-1/4", + "basis-2/4", + "basis-3/4", + "basis-1/5", + "basis-2/5", + "basis-3/5", + "basis-4/5", + "basis-1/6", + "basis-2/6", + "basis-3/6", + "basis-4/6", + "basis-5/6", + "basis-1/12", + "basis-2/12", + "basis-3/12", + "basis-4/12", + "basis-5/12", + "basis-6/12", + "basis-7/12", + "basis-8/12", + "basis-9/12", + "basis-10/12", + "basis-11/12", + "basis-full", + "flex-row", + "flex-row-reverse", + "flex-col", + "flex-col-reverse", + "flex-wrap", + "flex-wrap-reverse", + "flex-nowrap", + "flex-1", + "flex-auto", + "flex-initial", + "flex-none", + "grow", + "grow-0", + "shrink", + "shrink-0", + "order-1", + "order-2", + "order-3", + "order-4", + "order-5", + "order-6", + "order-7", + "order-8", + "order-9", + "order-10", + "order-11", + "order-12", + "order-first", + "order-last", + "order-none", + "grid-cols-1", + "grid-cols-2", + "grid-cols-3", + "grid-cols-4", + "grid-cols-5", + "grid-cols-6", + "grid-cols-7", + "grid-cols-8", + "grid-cols-9", + "grid-cols-10", + "grid-cols-11", + "grid-cols-12", + "grid-cols-none", + "grid-cols-subgrid", + "col-auto", + "col-span-1", + "col-span-2", + "col-span-3", + "col-span-4", + "col-span-5", + "col-span-6", + "col-span-7", + "col-span-8", + "col-span-9", + "col-span-10", + "col-span-11", + "col-span-12", + "col-span-full", + "col-start-1", + "col-start-2", + "col-start-3", + "col-start-4", + "col-start-5", + "col-start-6", + "col-start-7", + "col-start-8", + "col-start-9", + "col-start-10", + "col-start-11", + "col-start-12", + "col-start-13", + "col-start-auto", + "col-end-1", + "col-end-2", + "col-end-3", + "col-end-4", + "col-end-5", + "col-end-6", + "col-end-7", + "col-end-8", + "col-end-9", + "col-end-10", + "col-end-11", + "col-end-12", + "col-end-13", + "col-end-auto", + "grid-rows-1", + "grid-rows-2", + "grid-rows-3", + "grid-rows-4", + "grid-rows-5", + "grid-rows-6", + "grid-rows-7", + "grid-rows-8", + "grid-rows-9", + "grid-rows-10", + "grid-rows-11", + "grid-rows-12", + "grid-rows-none", + "grid-rows-subgrid", + "row-auto", + "row-span-1", + "row-span-2", + "row-span-3", + "row-span-4", + "row-span-5", + "row-span-6", + "row-span-7", + "row-span-8", + "row-span-9", + "row-span-10", + "row-span-11", + "row-span-12", + "row-span-full", + "row-start-1", + "row-start-2", + "row-start-3", + "row-start-4", + "row-start-5", + "row-start-6", + "row-start-7", + "row-start-8", + "row-start-9", + "row-start-10", + "row-start-11", + "row-start-12", + "row-start-13", + "row-start-auto", + "row-end-1", + "row-end-2", + "row-end-3", + "row-end-4", + "row-end-5", + "row-end-6", + "row-end-7", + "row-end-8", + "row-end-9", + "row-end-10", + "row-end-11", + "row-end-12", + "row-end-13", + "row-end-auto", + "grid-flow-row", + "grid-flow-col", + "grid-flow-dense", + "grid-flow-row-dense", + "grid-flow-col-dense", + "auto-cols-auto", + "auto-cols-min", + "auto-cols-max", + "auto-cols-fr", + "auto-rows-auto", + "auto-rows-min", + "auto-rows-max", + "auto-rows-fr", + "gap-0", + "gap-x-0", + "gap-y-0", + "gap-px", + "gap-x-px", + "gap-y-px", + "gap-0.5", + "gap-x-0.5", + "gap-y-0.5", + "gap-1", + "gap-x-1", + "gap-y-1", + "gap-1.5", + "gap-x-1.5", + "gap-y-1.5", + "gap-2", + "gap-x-2", + "gap-y-2", + "gap-2.5", + "gap-x-2.5", + "gap-y-2.5", + "gap-3", + "gap-x-3", + "gap-y-3", + "gap-3.5", + "gap-x-3.5", + "gap-y-3.5", + "gap-4", + "gap-x-4", + "gap-y-4", + "gap-5", + "gap-x-5", + "gap-y-5", + "gap-6", + "gap-x-6", + "gap-y-6", + "gap-7", + "gap-x-7", + "gap-y-7", + "gap-8", + "gap-x-8", + "gap-y-8", + "gap-9", + "gap-x-9", + "gap-y-9", + "gap-10", + "gap-x-10", + "gap-y-10", + "gap-11", + "gap-x-11", + "gap-y-11", + "gap-12", + "gap-x-12", + "gap-y-12", + "gap-14", + "gap-x-14", + "gap-y-14", + "gap-16", + "gap-x-16", + "gap-y-16", + "gap-20", + "gap-x-20", + "gap-y-20", + "gap-24", + "gap-x-24", + "gap-y-24", + "gap-28", + "gap-x-28", + "gap-y-28", + "gap-32", + "gap-x-32", + "gap-y-32", + "gap-36", + "gap-x-36", + "gap-y-36", + "gap-40", + "gap-x-40", + "gap-y-40", + "gap-44", + "gap-x-44", + "gap-y-44", + "gap-48", + "gap-x-48", + "gap-y-48", + "gap-52", + "gap-x-52", + "gap-y-52", + "gap-56", + "gap-x-56", + "gap-y-56", + "gap-60", + "gap-x-60", + "gap-y-60", + "gap-64", + "gap-x-64", + "gap-y-64", + "gap-72", + "gap-x-72", + "gap-y-72", + "gap-80", + "gap-x-80", + "gap-y-80", + "gap-96", + "gap-x-96", + "gap-y-96", + "justify-normal", + "justify-start", + "justify-end", + "justify-center", + "justify-between", + "justify-around", + "justify-evenly", + "justify-stretch", + "justify-items-start", + "justify-items-end", + "justify-items-center", + "justify-items-stretch", + "justify-self-auto", + "justify-self-start", + "justify-self-end", + "justify-self-center", + "justify-self-stretch", + "content-normal", + "content-center", + "content-start", + "content-end", + "content-between", + "content-around", + "content-evenly", + "content-baseline", + "content-stretch", + "items-start", + "items-end", + "items-center", + "items-baseline", + "items-stretch", + "self-auto", + "self-start", + "self-end", + "self-center", + "self-stretch", + "self-baseline", + "place-content-center", + "place-content-start", + "place-content-end", + "place-content-between", + "place-content-around", + "place-content-evenly", + "place-content-baseline", + "place-content-stretch", + "place-items-start", + "place-items-end", + "place-items-center", + "place-items-baseline", + "place-items-stretch", + "place-self-auto", + "place-self-start", + "place-self-end", + "place-self-center", + "place-self-stretch", + "p-0", + "px-0", + "py-0", + "ps-0", + "pe-0", + "pt-0", + "pr-0", + "pb-0", + "pl-0", + "p-px", + "px-px", + "py-px", + "ps-px", + "pe-px", + "pt-px", + "pr-px", + "pb-px", + "pl-px", + "p-0.5", + "px-0.5", + "py-0.5", + "ps-0.5", + "pe-0.5", + "pt-0.5", + "pr-0.5", + "pb-0.5", + "pl-0.5", + "p-1", + "px-1", + "py-1", + "ps-1", + "pe-1", + "pt-1", + "pr-1", + "pb-1", + "pl-1", + "p-1.5", + "px-1.5", + "py-1.5", + "ps-1.5", + "pe-1.5", + "pt-1.5", + "pr-1.5", + "pb-1.5", + "pl-1.5", + "p-2", + "px-2", + "py-2", + "ps-2", + "pe-2", + "pt-2", + "pr-2", + "pb-2", + "pl-2", + "p-2.5", + "px-2.5", + "py-2.5", + "ps-2.5", + "pe-2.5", + "pt-2.5", + "pr-2.5", + "pb-2.5", + "pl-2.5", + "p-3", + "px-3", + "py-3", + "ps-3", + "pe-3", + "pt-3", + "pr-3", + "pb-3", + "pl-3", + "p-3.5", + "px-3.5", + "py-3.5", + "ps-3.5", + "pe-3.5", + "pt-3.5", + "pr-3.5", + "pb-3.5", + "pl-3.5", + "p-4", + "px-4", + "py-4", + "ps-4", + "pe-4", + "pt-4", + "pr-4", + "pb-4", + "pl-4", + "p-5", + "px-5", + "py-5", + "ps-5", + "pe-5", + "pt-5", + "pr-5", + "pb-5", + "pl-5", + "p-6", + "px-6", + "py-6", + "ps-6", + "pe-6", + "pt-6", + "pr-6", + "pb-6", + "pl-6", + "p-7", + "px-7", + "py-7", + "ps-7", + "pe-7", + "pt-7", + "pr-7", + "pb-7", + "pl-7", + "p-8", + "px-8", + "py-8", + "ps-8", + "pe-8", + "pt-8", + "pr-8", + "pb-8", + "pl-8", + "p-9", + "px-9", + "py-9", + "ps-9", + "pe-9", + "pt-9", + "pr-9", + "pb-9", + "pl-9", + "p-10", + "px-10", + "py-10", + "ps-10", + "pe-10", + "pt-10", + "pr-10", + "pb-10", + "pl-10", + "p-11", + "px-11", + "py-11", + "ps-11", + "pe-11", + "pt-11", + "pr-11", + "pb-11", + "pl-11", + "p-12", + "px-12", + "py-12", + "ps-12", + "pe-12", + "pt-12", + "pr-12", + "pb-12", + "pl-12", + "p-14", + "px-14", + "py-14", + "ps-14", + "pe-14", + "pt-14", + "pr-14", + "pb-14", + "pl-14", + "p-16", + "px-16", + "py-16", + "ps-16", + "pe-16", + "pt-16", + "pr-16", + "pb-16", + "pl-16", + "p-20", + "px-20", + "py-20", + "ps-20", + "pe-20", + "pt-20", + "pr-20", + "pb-20", + "pl-20", + "p-24", + "px-24", + "py-24", + "ps-24", + "pe-24", + "pt-24", + "pr-24", + "pb-24", + "pl-24", + "p-28", + "px-28", + "py-28", + "ps-28", + "pe-28", + "pt-28", + "pr-28", + "pb-28", + "pl-28", + "p-32", + "px-32", + "py-32", + "ps-32", + "pe-32", + "pt-32", + "pr-32", + "pb-32", + "pl-32", + "p-36", + "px-36", + "py-36", + "ps-36", + "pe-36", + "pt-36", + "pr-36", + "pb-36", + "pl-36", + "p-40", + "px-40", + "py-40", + "ps-40", + "pe-40", + "pt-40", + "pr-40", + "pb-40", + "pl-40", + "p-44", + "px-44", + "py-44", + "ps-44", + "pe-44", + "pt-44", + "pr-44", + "pb-44", + "pl-44", + "p-48", + "px-48", + "py-48", + "ps-48", + "pe-48", + "pt-48", + "pr-48", + "pb-48", + "pl-48", + "p-52", + "px-52", + "py-52", + "ps-52", + "pe-52", + "pt-52", + "pr-52", + "pb-52", + "pl-52", + "p-56", + "px-56", + "py-56", + "ps-56", + "pe-56", + "pt-56", + "pr-56", + "pb-56", + "pl-56", + "p-60", + "px-60", + "py-60", + "ps-60", + "pe-60", + "pt-60", + "pr-60", + "pb-60", + "pl-60", + "p-64", + "px-64", + "py-64", + "ps-64", + "pe-64", + "pt-64", + "pr-64", + "pb-64", + "pl-64", + "p-72", + "px-72", + "py-72", + "ps-72", + "pe-72", + "pt-72", + "pr-72", + "pb-72", + "pl-72", + "p-80", + "px-80", + "py-80", + "ps-80", + "pe-80", + "pt-80", + "pr-80", + "pb-80", + "pl-80", + "p-96", + "px-96", + "py-96", + "ps-96", + "pe-96", + "pt-96", + "pr-96", + "pb-96", + "pl-96", + "m-0", + "mx-0", + "my-0", + "ms-0", + "me-0", + "mt-0", + "mr-0", + "mb-0", + "ml-0", + "m-px", + "mx-px", + "my-px", + "ms-px", + "me-px", + "mt-px", + "mr-px", + "mb-px", + "ml-px", + "m-0.5", + "mx-0.5", + "my-0.5", + "ms-0.5", + "me-0.5", + "mt-0.5", + "mr-0.5", + "mb-0.5", + "ml-0.5", + "m-1", + "mx-1", + "my-1", + "ms-1", + "me-1", + "mt-1", + "mr-1", + "mb-1", + "ml-1", + "m-1.5", + "mx-1.5", + "my-1.5", + "ms-1.5", + "me-1.5", + "mt-1.5", + "mr-1.5", + "mb-1.5", + "ml-1.5", + "m-2", + "mx-2", + "my-2", + "ms-2", + "me-2", + "mt-2", + "mr-2", + "mb-2", + "ml-2", + "m-2.5", + "mx-2.5", + "my-2.5", + "ms-2.5", + "me-2.5", + "mt-2.5", + "mr-2.5", + "mb-2.5", + "ml-2.5", + "m-3", + "mx-3", + "my-3", + "ms-3", + "me-3", + "mt-3", + "mr-3", + "mb-3", + "ml-3", + "m-3.5", + "mx-3.5", + "my-3.5", + "ms-3.5", + "me-3.5", + "mt-3.5", + "mr-3.5", + "mb-3.5", + "ml-3.5", + "m-4", + "mx-4", + "my-4", + "ms-4", + "me-4", + "mt-4", + "mr-4", + "mb-4", + "ml-4", + "m-5", + "mx-5", + "my-5", + "ms-5", + "me-5", + "mt-5", + "mr-5", + "mb-5", + "ml-5", + "m-6", + "mx-6", + "my-6", + "ms-6", + "me-6", + "mt-6", + "mr-6", + "mb-6", + "ml-6", + "m-7", + "mx-7", + "my-7", + "ms-7", + "me-7", + "mt-7", + "mr-7", + "mb-7", + "ml-7", + "m-8", + "mx-8", + "my-8", + "ms-8", + "me-8", + "mt-8", + "mr-8", + "mb-8", + "ml-8", + "m-9", + "mx-9", + "my-9", + "ms-9", + "me-9", + "mt-9", + "mr-9", + "mb-9", + "ml-9", + "m-10", + "mx-10", + "my-10", + "ms-10", + "me-10", + "mt-10", + "mr-10", + "mb-10", + "ml-10", + "m-11", + "mx-11", + "my-11", + "ms-11", + "me-11", + "mt-11", + "mr-11", + "mb-11", + "ml-11", + "m-12", + "mx-12", + "my-12", + "ms-12", + "me-12", + "mt-12", + "mr-12", + "mb-12", + "ml-12", + "m-14", + "mx-14", + "my-14", + "ms-14", + "me-14", + "mt-14", + "mr-14", + "mb-14", + "ml-14", + "m-16", + "mx-16", + "my-16", + "ms-16", + "me-16", + "mt-16", + "mr-16", + "mb-16", + "ml-16", + "m-20", + "mx-20", + "my-20", + "ms-20", + "me-20", + "mt-20", + "mr-20", + "mb-20", + "ml-20", + "m-24", + "mx-24", + "my-24", + "ms-24", + "me-24", + "mt-24", + "mr-24", + "mb-24", + "ml-24", + "m-28", + "mx-28", + "my-28", + "ms-28", + "me-28", + "mt-28", + "mr-28", + "mb-28", + "ml-28", + "m-32", + "mx-32", + "my-32", + "ms-32", + "me-32", + "mt-32", + "mr-32", + "mb-32", + "ml-32", + "m-36", + "mx-36", + "my-36", + "ms-36", + "me-36", + "mt-36", + "mr-36", + "mb-36", + "ml-36", + "m-40", + "mx-40", + "my-40", + "ms-40", + "me-40", + "mt-40", + "mr-40", + "mb-40", + "ml-40", + "m-44", + "mx-44", + "my-44", + "ms-44", + "me-44", + "mt-44", + "mr-44", + "mb-44", + "ml-44", + "m-48", + "mx-48", + "my-48", + "ms-48", + "me-48", + "mt-48", + "mr-48", + "mb-48", + "ml-48", + "m-52", + "mx-52", + "my-52", + "ms-52", + "me-52", + "mt-52", + "mr-52", + "mb-52", + "ml-52", + "m-56", + "mx-56", + "my-56", + "ms-56", + "me-56", + "mt-56", + "mr-56", + "mb-56", + "ml-56", + "m-60", + "mx-60", + "my-60", + "ms-60", + "me-60", + "mt-60", + "mr-60", + "mb-60", + "ml-60", + "m-64", + "mx-64", + "my-64", + "ms-64", + "me-64", + "mt-64", + "mr-64", + "mb-64", + "ml-64", + "m-72", + "mx-72", + "my-72", + "ms-72", + "me-72", + "mt-72", + "mr-72", + "mb-72", + "ml-72", + "m-80", + "mx-80", + "my-80", + "ms-80", + "me-80", + "mt-80", + "mr-80", + "mb-80", + "ml-80", + "m-96", + "mx-96", + "my-96", + "ms-96", + "me-96", + "mt-96", + "mr-96", + "mb-96", + "ml-96", + "m-auto", + "mx-auto", + "my-auto", + "ms-auto", + "me-auto", + "mt-auto", + "mr-auto", + "mb-auto", + "ml-auto", + "space-x-0", + "space-y-0", + "space-x-0.5", + "space-y-0.5", + "space-x-1", + "space-y-1", + "space-x-1.5", + "space-y-1.5", + "space-x-2", + "space-y-2", + "space-x-2.5", + "space-y-2.5", + "space-x-3", + "space-y-3", + "space-x-3.5", + "space-y-3.5", + "space-x-4", + "space-y-4", + "space-x-5", + "space-y-5", + "space-x-6", + "space-y-6", + "space-x-7", + "space-y-7", + "space-x-8", + "space-y-8", + "space-x-9", + "space-y-9", + "space-x-10", + "space-y-10", + "space-x-11", + "space-y-11", + "space-x-12", + "space-y-12", + "space-x-14", + "space-y-14", + "space-x-16", + "space-y-16", + "space-x-20", + "space-y-20", + "space-x-24", + "space-y-24", + "space-x-28", + "space-y-28", + "space-x-32", + "space-y-32", + "space-x-36", + "space-y-36", + "space-x-40", + "space-y-40", + "space-x-44", + "space-y-44", + "space-x-48", + "space-y-48", + "space-x-52", + "space-y-52", + "space-x-56", + "space-y-56", + "space-x-60", + "space-y-60", + "space-x-64", + "space-y-64", + "space-x-72", + "space-y-72", + "space-x-80", + "space-y-80", + "space-x-96", + "space-y-96", + "space-x-px", + "space-y-px", + "space-y-reverse", + "space-x-reverse", + "w-0", + "w-px", + "w-0.5", + "w-1", + "w-1.5", + "w-2", + "w-2.5", + "w-3", + "w-3.5", + "w-4", + "w-5", + "w-6", + "w-7", + "w-8", + "w-9", + "w-10", + "w-11", + "w-12", + "w-14", + "w-16", + "w-20", + "w-24", + "w-28", + "w-32", + "w-36", + "w-40", + "w-44", + "w-48", + "w-52", + "w-56", + "w-60", + "w-64", + "w-72", + "w-80", + "w-96", + "w-auto", + "w-1/2", + "w-1/3", + "w-2/3", + "w-1/4", + "w-2/4", + "w-3/4", + "w-1/5", + "w-2/5", + "w-3/5", + "w-4/5", + "w-1/6", + "w-2/6", + "w-3/6", + "w-4/6", + "w-5/6", + "w-1/12", + "w-2/12", + "w-3/12", + "w-4/12", + "w-5/12", + "w-6/12", + "w-7/12", + "w-8/12", + "w-9/12", + "w-10/12", + "w-11/12", + "w-full", + "w-screen", + "w-svw", + "w-lvw", + "w-dvw", + "w-min", + "w-max", + "w-fit", + "min-w-0", + "min-w-1", + "min-w-2", + "min-w-3", + "min-w-4", + "min-w-5", + "min-w-6", + "min-w-7", + "min-w-8", + "min-w-9", + "min-w-10", + "min-w-11", + "min-w-12", + "min-w-14", + "min-w-16", + "min-w-20", + "min-w-24", + "min-w-28", + "min-w-32", + "min-w-36", + "min-w-40", + "min-w-44", + "min-w-48", + "min-w-52", + "min-w-56", + "min-w-60", + "min-w-64", + "min-w-72", + "min-w-80", + "min-w-96", + "min-w-px", + "min-w-0.5", + "min-w-1.5", + "min-w-2.5", + "min-w-3.5", + "min-w-full", + "min-w-min", + "min-w-max", + "min-w-fit", + "max-w-0", + "max-w-px", + "max-w-0.5", + "max-w-1", + "max-w-1.5", + "max-w-2", + "max-w-2.5", + "max-w-3", + "max-w-3.5", + "max-w-4", + "max-w-5", + "max-w-6", + "max-w-7", + "max-w-8", + "max-w-9", + "max-w-10", + "max-w-11", + "max-w-12", + "max-w-14", + "max-w-16", + "max-w-20", + "max-w-24", + "max-w-28", + "max-w-32", + "max-w-36", + "max-w-40", + "max-w-44", + "max-w-48", + "max-w-52", + "max-w-56", + "max-w-60", + "max-w-64", + "max-w-72", + "max-w-80", + "max-w-96", + "max-w-none", + "max-w-xs", + "max-w-sm", + "max-w-md", + "max-w-lg", + "max-w-xl", + "max-w-2xl", + "max-w-3xl", + "max-w-4xl", + "max-w-5xl", + "max-w-6xl", + "max-w-7xl", + "max-w-full", + "max-w-min", + "max-w-max", + "max-w-fit", + "max-w-prose", + "max-w-screen-sm", + "max-w-screen-md", + "max-w-screen-lg", + "max-w-screen-xl", + "max-w-screen-2xl", + "h-0", + "h-px", + "h-0.5", + "h-1", + "h-1.5", + "h-2", + "h-2.5", + "h-3", + "h-3.5", + "h-4", + "h-5", + "h-6", + "h-7", + "h-8", + "h-9", + "h-10", + "h-11", + "h-12", + "h-14", + "h-16", + "h-20", + "h-24", + "h-28", + "h-32", + "h-36", + "h-40", + "h-44", + "h-48", + "h-52", + "h-56", + "h-60", + "h-64", + "h-72", + "h-80", + "h-96", + "h-auto", + "h-1/2", + "h-1/3", + "h-2/3", + "h-1/4", + "h-2/4", + "h-3/4", + "h-1/5", + "h-2/5", + "h-3/5", + "h-4/5", + "h-1/6", + "h-2/6", + "h-3/6", + "h-4/6", + "h-5/6", + "h-full", + "h-screen", + "h-svh", + "h-lvh", + "h-dvh", + "h-min", + "h-max", + "h-fit", + "min-h-0", + "min-h-1", + "min-h-2", + "min-h-3", + "min-h-4", + "min-h-5", + "min-h-6", + "min-h-7", + "min-h-8", + "min-h-9", + "min-h-10", + "min-h-11", + "min-h-12", + "min-h-14", + "min-h-16", + "min-h-20", + "min-h-24", + "min-h-28", + "min-h-32", + "min-h-36", + "min-h-40", + "min-h-44", + "min-h-48", + "min-h-52", + "min-h-56", + "min-h-60", + "min-h-64", + "min-h-72", + "min-h-80", + "min-h-96", + "min-h-px", + "min-h-0.5", + "min-h-1.5", + "min-h-2.5", + "min-h-3.5", + "min-h-full", + "min-h-screen", + "min-h-svh", + "min-h-lvh", + "min-h-dvh", + "min-h-min", + "min-h-max", + "min-h-fit", + "max-h-0", + "max-h-px", + "max-h-0.5", + "max-h-1", + "max-h-1.5", + "max-h-2", + "max-h-2.5", + "max-h-3", + "max-h-3.5", + "max-h-4", + "max-h-5", + "max-h-6", + "max-h-7", + "max-h-8", + "max-h-9", + "max-h-10", + "max-h-11", + "max-h-12", + "max-h-14", + "max-h-16", + "max-h-20", + "max-h-24", + "max-h-28", + "max-h-32", + "max-h-36", + "max-h-40", + "max-h-44", + "max-h-48", + "max-h-52", + "max-h-56", + "max-h-60", + "max-h-64", + "max-h-72", + "max-h-80", + "max-h-96", + "max-h-none", + "max-h-full", + "max-h-screen", + "max-h-svh", + "max-h-lvh", + "max-h-dvh", + "max-h-min", + "max-h-max", + "max-h-fit", + "size-0", + "size-px", + "size-0.5", + "size-1", + "size-1.5", + "size-2", + "size-2.5", + "size-3", + "size-3.5", + "size-4", + "size-5", + "size-6", + "size-7", + "size-8", + "size-9", + "size-10", + "size-11", + "size-12", + "size-14", + "size-16", + "size-20", + "size-24", + "size-28", + "size-32", + "size-36", + "size-40", + "size-44", + "size-48", + "size-52", + "size-56", + "size-60", + "size-64", + "size-72", + "size-80", + "size-96", + "size-auto", + "size-1/2", + "size-1/3", + "size-2/3", + "size-1/4", + "size-2/4", + "size-3/4", + "size-1/5", + "size-2/5", + "size-3/5", + "size-4/5", + "size-1/6", + "size-2/6", + "size-3/6", + "size-4/6", + "size-5/6", + "size-1/12", + "size-2/12", + "size-3/12", + "size-4/12", + "size-5/12", + "size-6/12", + "size-7/12", + "size-8/12", + "size-9/12", + "size-10/12", + "size-11/12", + "size-full", + "size-min", + "size-max", + "size-fit", + "font-sans", + "font-serif", + "font-mono", + "text-xs", + "text-sm", + "text-base", + "text-lg", + "text-xl", + "text-2xl", + "text-3xl", + "text-4xl", + "text-5xl", + "text-6xl", + "text-7xl", + "text-8xl", + "text-9xl", + "antialiased", + "subpixel-antialiased", + "italic", + "not-italic", + "font-thin", + "font-extralight", + "font-light", + "font-normal", + "font-medium", + "font-semibold", + "font-bold", + "font-extrabold", + "font-black", + "normal-nums", + "ordinal", + "slashed-zero", + "lining-nums", + "oldstyle-nums", + "proportional-nums", + "tabular-nums", + "diagonal-fractions", + "stacked-fractions", + "tracking-tighter", + "tracking-tight", + "tracking-normal", + "tracking-wide", + "tracking-wider", + "tracking-widest", + "line-clamp-1", + "line-clamp-2", + "line-clamp-3", + "line-clamp-4", + "line-clamp-5", + "line-clamp-6", + "line-clamp-none", + "leading-3", + "leading-4", + "leading-5", + "leading-6", + "leading-7", + "leading-8", + "leading-9", + "leading-10", + "leading-none", + "leading-tight", + "leading-snug", + "leading-normal", + "leading-relaxed", + "leading-loose", + "list-image-none", + "list-inside", + "list-outside", + "list-none", + "list-disc", + "list-decimal", + "text-left", + "text-center", + "text-right", + "text-justify", + "text-start", + "text-end", + "text-inherit", + "text-current", + "text-transparent", + "text-black", + "text-white", + "text-slate-50", + "text-slate-100", + "text-slate-200", + "text-slate-300", + "text-slate-400", + "text-slate-500", + "text-slate-600", + "text-slate-700", + "text-slate-800", + "text-slate-900", + "text-slate-950", + "text-gray-50", + "text-gray-100", + "text-gray-200", + "text-gray-300", + "text-gray-400", + "text-gray-500", + "text-gray-600", + "text-gray-700", + "text-gray-800", + "text-gray-900", + "text-gray-950", + "text-zinc-50", + "text-zinc-100", + "text-zinc-200", + "text-zinc-300", + "text-zinc-400", + "text-zinc-500", + "text-zinc-600", + "text-zinc-700", + "text-zinc-800", + "text-zinc-900", + "text-zinc-950", + "text-neutral-50", + "text-neutral-100", + "text-neutral-200", + "text-neutral-300", + "text-neutral-400", + "text-neutral-500", + "text-neutral-600", + "text-neutral-700", + "text-neutral-800", + "text-neutral-900", + "text-neutral-950", + "text-stone-50", + "text-stone-100", + "text-stone-200", + "text-stone-300", + "text-stone-400", + "text-stone-500", + "text-stone-600", + "text-stone-700", + "text-stone-800", + "text-stone-900", + "text-stone-950", + "text-red-50", + "text-red-100", + "text-red-200", + "text-red-300", + "text-red-400", + "text-red-500", + "text-red-600", + "text-red-700", + "text-red-800", + "text-red-900", + "text-red-950", + "text-orange-50", + "text-orange-100", + "text-orange-200", + "text-orange-300", + "text-orange-400", + "text-orange-500", + "text-orange-600", + "text-orange-700", + "text-orange-800", + "text-orange-900", + "text-orange-950", + "text-amber-50", + "text-amber-100", + "text-amber-200", + "text-amber-300", + "text-amber-400", + "text-amber-500", + "text-amber-600", + "text-amber-700", + "text-amber-800", + "text-amber-900", + "text-amber-950", + "text-yellow-50", + "text-yellow-100", + "text-yellow-200", + "text-yellow-300", + "text-yellow-400", + "text-yellow-500", + "text-yellow-600", + "text-yellow-700", + "text-yellow-800", + "text-yellow-900", + "text-yellow-950", + "text-lime-50", + "text-lime-100", + "text-lime-200", + "text-lime-300", + "text-lime-400", + "text-lime-500", + "text-lime-600", + "text-lime-700", + "text-lime-800", + "text-lime-900", + "text-lime-950", + "text-green-50", + "text-green-100", + "text-green-200", + "text-green-300", + "text-green-400", + "text-green-500", + "text-green-600", + "text-green-700", + "text-green-800", + "text-green-900", + "text-green-950", + "text-emerald-50", + "text-emerald-100", + "text-emerald-200", + "text-emerald-300", + "text-emerald-400", + "text-emerald-500", + "text-emerald-600", + "text-emerald-700", + "text-emerald-800", + "text-emerald-900", + "text-emerald-950", + "text-teal-50", + "text-teal-100", + "text-teal-200", + "text-teal-300", + "text-teal-400", + "text-teal-500", + "text-teal-600", + "text-teal-700", + "text-teal-800", + "text-teal-900", + "text-teal-950", + "text-cyan-50", + "text-cyan-100", + "text-cyan-200", + "text-cyan-300", + "text-cyan-400", + "text-cyan-500", + "text-cyan-600", + "text-cyan-700", + "text-cyan-800", + "text-cyan-900", + "text-cyan-950", + "text-sky-50", + "text-sky-100", + "text-sky-200", + "text-sky-300", + "text-sky-400", + "text-sky-500", + "text-sky-600", + "text-sky-700", + "text-sky-800", + "text-sky-900", + "text-sky-950", + "text-blue-50", + "text-blue-100", + "text-blue-200", + "text-blue-300", + "text-blue-400", + "text-blue-500", + "text-blue-600", + "text-blue-700", + "text-blue-800", + "text-blue-900", + "text-blue-950", + "text-indigo-50", + "text-indigo-100", + "text-indigo-200", + "text-indigo-300", + "text-indigo-400", + "text-indigo-500", + "text-indigo-600", + "text-indigo-700", + "text-indigo-800", + "text-indigo-900", + "text-indigo-950", + "text-violet-50", + "text-violet-100", + "text-violet-200", + "text-violet-300", + "text-violet-400", + "text-violet-500", + "text-violet-600", + "text-violet-700", + "text-violet-800", + "text-violet-900", + "text-violet-950", + "text-purple-50", + "text-purple-100", + "text-purple-200", + "text-purple-300", + "text-purple-400", + "text-purple-500", + "text-purple-600", + "text-purple-700", + "text-purple-800", + "text-purple-900", + "text-purple-950", + "text-fuchsia-50", + "text-fuchsia-100", + "text-fuchsia-200", + "text-fuchsia-300", + "text-fuchsia-400", + "text-fuchsia-500", + "text-fuchsia-600", + "text-fuchsia-700", + "text-fuchsia-800", + "text-fuchsia-900", + "text-fuchsia-950", + "text-pink-50", + "text-pink-100", + "text-pink-200", + "text-pink-300", + "text-pink-400", + "text-pink-500", + "text-pink-600", + "text-pink-700", + "text-pink-800", + "text-pink-900", + "text-pink-950", + "text-rose-50", + "text-rose-100", + "text-rose-200", + "text-rose-300", + "text-rose-400", + "text-rose-500", + "text-rose-600", + "text-rose-700", + "text-rose-800", + "text-rose-900", + "text-rose-950", + "underline", + "overline", + "line-through", + "no-underline", + "decoration-inherit", + "decoration-current", + "decoration-transparent", + "decoration-black", + "decoration-white", + "decoration-slate-50", + "decoration-slate-100", + "decoration-slate-200", + "decoration-slate-300", + "decoration-slate-400", + "decoration-slate-500", + "decoration-slate-600", + "decoration-slate-700", + "decoration-slate-800", + "decoration-slate-900", + "decoration-slate-950", + "decoration-gray-50", + "decoration-gray-100", + "decoration-gray-200", + "decoration-gray-300", + "decoration-gray-400", + "decoration-gray-500", + "decoration-gray-600", + "decoration-gray-700", + "decoration-gray-800", + "decoration-gray-900", + "decoration-gray-950", + "decoration-zinc-50", + "decoration-zinc-100", + "decoration-zinc-200", + "decoration-zinc-300", + "decoration-zinc-400", + "decoration-zinc-500", + "decoration-zinc-600", + "decoration-zinc-700", + "decoration-zinc-800", + "decoration-zinc-900", + "decoration-zinc-950", + "decoration-neutral-50", + "decoration-neutral-100", + "decoration-neutral-200", + "decoration-neutral-300", + "decoration-neutral-400", + "decoration-neutral-500", + "decoration-neutral-600", + "decoration-neutral-700", + "decoration-neutral-800", + "decoration-neutral-900", + "decoration-neutral-950", + "decoration-stone-50", + "decoration-stone-100", + "decoration-stone-200", + "decoration-stone-300", + "decoration-stone-400", + "decoration-stone-500", + "decoration-stone-600", + "decoration-stone-700", + "decoration-stone-800", + "decoration-stone-900", + "decoration-stone-950", + "decoration-red-50", + "decoration-red-100", + "decoration-red-200", + "decoration-red-300", + "decoration-red-400", + "decoration-red-500", + "decoration-red-600", + "decoration-red-700", + "decoration-red-800", + "decoration-red-900", + "decoration-red-950", + "decoration-orange-50", + "decoration-orange-100", + "decoration-orange-200", + "decoration-orange-300", + "decoration-orange-400", + "decoration-orange-500", + "decoration-orange-600", + "decoration-orange-700", + "decoration-orange-800", + "decoration-orange-900", + "decoration-orange-950", + "decoration-amber-50", + "decoration-amber-100", + "decoration-amber-200", + "decoration-amber-300", + "decoration-amber-400", + "decoration-amber-500", + "decoration-amber-600", + "decoration-amber-700", + "decoration-amber-800", + "decoration-amber-900", + "decoration-amber-950", + "decoration-yellow-50", + "decoration-yellow-100", + "decoration-yellow-200", + "decoration-yellow-300", + "decoration-yellow-400", + "decoration-yellow-500", + "decoration-yellow-600", + "decoration-yellow-700", + "decoration-yellow-800", + "decoration-yellow-900", + "decoration-yellow-950", + "decoration-lime-50", + "decoration-lime-100", + "decoration-lime-200", + "decoration-lime-300", + "decoration-lime-400", + "decoration-lime-500", + "decoration-lime-600", + "decoration-lime-700", + "decoration-lime-800", + "decoration-lime-900", + "decoration-lime-950", + "decoration-green-50", + "decoration-green-100", + "decoration-green-200", + "decoration-green-300", + "decoration-green-400", + "decoration-green-500", + "decoration-green-600", + "decoration-green-700", + "decoration-green-800", + "decoration-green-900", + "decoration-green-950", + "decoration-emerald-50", + "decoration-emerald-100", + "decoration-emerald-200", + "decoration-emerald-300", + "decoration-emerald-400", + "decoration-emerald-500", + "decoration-emerald-600", + "decoration-emerald-700", + "decoration-emerald-800", + "decoration-emerald-900", + "decoration-emerald-950", + "decoration-teal-50", + "decoration-teal-100", + "decoration-teal-200", + "decoration-teal-300", + "decoration-teal-400", + "decoration-teal-500", + "decoration-teal-600", + "decoration-teal-700", + "decoration-teal-800", + "decoration-teal-900", + "decoration-teal-950", + "decoration-cyan-50", + "decoration-cyan-100", + "decoration-cyan-200", + "decoration-cyan-300", + "decoration-cyan-400", + "decoration-cyan-500", + "decoration-cyan-600", + "decoration-cyan-700", + "decoration-cyan-800", + "decoration-cyan-900", + "decoration-cyan-950", + "decoration-sky-50", + "decoration-sky-100", + "decoration-sky-200", + "decoration-sky-300", + "decoration-sky-400", + "decoration-sky-500", + "decoration-sky-600", + "decoration-sky-700", + "decoration-sky-800", + "decoration-sky-900", + "decoration-sky-950", + "decoration-blue-50", + "decoration-blue-100", + "decoration-blue-200", + "decoration-blue-300", + "decoration-blue-400", + "decoration-blue-500", + "decoration-blue-600", + "decoration-blue-700", + "decoration-blue-800", + "decoration-blue-900", + "decoration-blue-950", + "decoration-indigo-50", + "decoration-indigo-100", + "decoration-indigo-200", + "decoration-indigo-300", + "decoration-indigo-400", + "decoration-indigo-500", + "decoration-indigo-600", + "decoration-indigo-700", + "decoration-indigo-800", + "decoration-indigo-900", + "decoration-indigo-950", + "decoration-violet-50", + "decoration-violet-100", + "decoration-violet-200", + "decoration-violet-300", + "decoration-violet-400", + "decoration-violet-500", + "decoration-violet-600", + "decoration-violet-700", + "decoration-violet-800", + "decoration-violet-900", + "decoration-violet-950", + "decoration-purple-50", + "decoration-purple-100", + "decoration-purple-200", + "decoration-purple-300", + "decoration-purple-400", + "decoration-purple-500", + "decoration-purple-600", + "decoration-purple-700", + "decoration-purple-800", + "decoration-purple-900", + "decoration-purple-950", + "decoration-fuchsia-50", + "decoration-fuchsia-100", + "decoration-fuchsia-200", + "decoration-fuchsia-300", + "decoration-fuchsia-400", + "decoration-fuchsia-500", + "decoration-fuchsia-600", + "decoration-fuchsia-700", + "decoration-fuchsia-800", + "decoration-fuchsia-900", + "decoration-fuchsia-950", + "decoration-pink-50", + "decoration-pink-100", + "decoration-pink-200", + "decoration-pink-300", + "decoration-pink-400", + "decoration-pink-500", + "decoration-pink-600", + "decoration-pink-700", + "decoration-pink-800", + "decoration-pink-900", + "decoration-pink-950", + "decoration-rose-50", + "decoration-rose-100", + "decoration-rose-200", + "decoration-rose-300", + "decoration-rose-400", + "decoration-rose-500", + "decoration-rose-600", + "decoration-rose-700", + "decoration-rose-800", + "decoration-rose-900", + "decoration-rose-950", + "decoration-solid", + "decoration-double", + "decoration-dotted", + "decoration-dashed", + "decoration-wavy", + "decoration-auto", + "decoration-from-font", + "decoration-0", + "decoration-1", + "decoration-2", + "decoration-4", + "decoration-8", + "underline-offset-auto", + "underline-offset-0", + "underline-offset-1", + "underline-offset-2", + "underline-offset-4", + "underline-offset-8", + "uppercase", + "lowercase", + "capitalize", + "normal-case", + "truncate", + "text-ellipsis", + "text-clip", + "text-wrap", + "text-nowrap", + "text-balance", + "text-pretty", + "indent-0", + "indent-px", + "indent-0.5", + "indent-1", + "indent-1.5", + "indent-2", + "indent-2.5", + "indent-3", + "indent-3.5", + "indent-4", + "indent-5", + "indent-6", + "indent-7", + "indent-8", + "indent-9", + "indent-10", + "indent-11", + "indent-12", + "indent-14", + "indent-16", + "indent-20", + "indent-24", + "indent-28", + "indent-32", + "indent-36", + "indent-40", + "indent-44", + "indent-48", + "indent-52", + "indent-56", + "indent-60", + "indent-64", + "indent-72", + "indent-80", + "indent-96", + "align-baseline", + "align-top", + "align-middle", + "align-bottom", + "align-text-top", + "align-text-bottom", + "align-sub", + "align-super", + "whitespace-normal", + "whitespace-nowrap", + "whitespace-pre", + "whitespace-pre-line", + "whitespace-pre-wrap", + "whitespace-break-spaces", + "break-normal", + "break-words", + "break-all", + "break-keep", + "hyphens-none", + "hyphens-manual", + "hyphens-auto", + "content-none", + "bg-fixed", + "bg-local", + "bg-scroll", + "bg-clip-border", + "bg-clip-padding", + "bg-clip-content", + "bg-clip-text", + "bg-inherit", + "bg-current", + "bg-transparent", + "bg-black", + "bg-white", + "bg-slate-50", + "bg-slate-100", + "bg-slate-200", + "bg-slate-300", + "bg-slate-400", + "bg-slate-500", + "bg-slate-600", + "bg-slate-700", + "bg-slate-800", + "bg-slate-900", + "bg-slate-950", + "bg-gray-50", + "bg-gray-100", + "bg-gray-200", + "bg-gray-300", + "bg-gray-400", + "bg-gray-500", + "bg-gray-600", + "bg-gray-700", + "bg-gray-800", + "bg-gray-900", + "bg-gray-950", + "bg-zinc-50", + "bg-zinc-100", + "bg-zinc-200", + "bg-zinc-300", + "bg-zinc-400", + "bg-zinc-500", + "bg-zinc-600", + "bg-zinc-700", + "bg-zinc-800", + "bg-zinc-900", + "bg-zinc-950", + "bg-neutral-50", + "bg-neutral-100", + "bg-neutral-200", + "bg-neutral-300", + "bg-neutral-400", + "bg-neutral-500", + "bg-neutral-600", + "bg-neutral-700", + "bg-neutral-800", + "bg-neutral-900", + "bg-neutral-950", + "bg-stone-50", + "bg-stone-100", + "bg-stone-200", + "bg-stone-300", + "bg-stone-400", + "bg-stone-500", + "bg-stone-600", + "bg-stone-700", + "bg-stone-800", + "bg-stone-900", + "bg-stone-950", + "bg-red-50", + "bg-red-100", + "bg-red-200", + "bg-red-300", + "bg-red-400", + "bg-red-500", + "bg-red-600", + "bg-red-700", + "bg-red-800", + "bg-red-900", + "bg-red-950", + "bg-orange-50", + "bg-orange-100", + "bg-orange-200", + "bg-orange-300", + "bg-orange-400", + "bg-orange-500", + "bg-orange-600", + "bg-orange-700", + "bg-orange-800", + "bg-orange-900", + "bg-orange-950", + "bg-amber-50", + "bg-amber-100", + "bg-amber-200", + "bg-amber-300", + "bg-amber-400", + "bg-amber-500", + "bg-amber-600", + "bg-amber-700", + "bg-amber-800", + "bg-amber-900", + "bg-amber-950", + "bg-yellow-50", + "bg-yellow-100", + "bg-yellow-200", + "bg-yellow-300", + "bg-yellow-400", + "bg-yellow-500", + "bg-yellow-600", + "bg-yellow-700", + "bg-yellow-800", + "bg-yellow-900", + "bg-yellow-950", + "bg-lime-50", + "bg-lime-100", + "bg-lime-200", + "bg-lime-300", + "bg-lime-400", + "bg-lime-500", + "bg-lime-600", + "bg-lime-700", + "bg-lime-800", + "bg-lime-900", + "bg-lime-950", + "bg-green-50", + "bg-green-100", + "bg-green-200", + "bg-green-300", + "bg-green-400", + "bg-green-500", + "bg-green-600", + "bg-green-700", + "bg-green-800", + "bg-green-900", + "bg-green-950", + "bg-emerald-50", + "bg-emerald-100", + "bg-emerald-200", + "bg-emerald-300", + "bg-emerald-400", + "bg-emerald-500", + "bg-emerald-600", + "bg-emerald-700", + "bg-emerald-800", + "bg-emerald-900", + "bg-emerald-950", + "bg-teal-50", + "bg-teal-100", + "bg-teal-200", + "bg-teal-300", + "bg-teal-400", + "bg-teal-500", + "bg-teal-600", + "bg-teal-700", + "bg-teal-800", + "bg-teal-900", + "bg-teal-950", + "bg-cyan-50", + "bg-cyan-100", + "bg-cyan-200", + "bg-cyan-300", + "bg-cyan-400", + "bg-cyan-500", + "bg-cyan-600", + "bg-cyan-700", + "bg-cyan-800", + "bg-cyan-900", + "bg-cyan-950", + "bg-sky-50", + "bg-sky-100", + "bg-sky-200", + "bg-sky-300", + "bg-sky-400", + "bg-sky-500", + "bg-sky-600", + "bg-sky-700", + "bg-sky-800", + "bg-sky-900", + "bg-sky-950", + "bg-blue-50", + "bg-blue-100", + "bg-blue-200", + "bg-blue-300", + "bg-blue-400", + "bg-blue-500", + "bg-blue-600", + "bg-blue-700", + "bg-blue-800", + "bg-blue-900", + "bg-blue-950", + "bg-indigo-50", + "bg-indigo-100", + "bg-indigo-200", + "bg-indigo-300", + "bg-indigo-400", + "bg-indigo-500", + "bg-indigo-600", + "bg-indigo-700", + "bg-indigo-800", + "bg-indigo-900", + "bg-indigo-950", + "bg-violet-50", + "bg-violet-100", + "bg-violet-200", + "bg-violet-300", + "bg-violet-400", + "bg-violet-500", + "bg-violet-600", + "bg-violet-700", + "bg-violet-800", + "bg-violet-900", + "bg-violet-950", + "bg-purple-50", + "bg-purple-100", + "bg-purple-200", + "bg-purple-300", + "bg-purple-400", + "bg-purple-500", + "bg-purple-600", + "bg-purple-700", + "bg-purple-800", + "bg-purple-900", + "bg-purple-950", + "bg-fuchsia-50", + "bg-fuchsia-100", + "bg-fuchsia-200", + "bg-fuchsia-300", + "bg-fuchsia-400", + "bg-fuchsia-500", + "bg-fuchsia-600", + "bg-fuchsia-700", + "bg-fuchsia-800", + "bg-fuchsia-900", + "bg-fuchsia-950", + "bg-pink-50", + "bg-pink-100", + "bg-pink-200", + "bg-pink-300", + "bg-pink-400", + "bg-pink-500", + "bg-pink-600", + "bg-pink-700", + "bg-pink-800", + "bg-pink-900", + "bg-pink-950", + "bg-rose-50", + "bg-rose-100", + "bg-rose-200", + "bg-rose-300", + "bg-rose-400", + "bg-rose-500", + "bg-rose-600", + "bg-rose-700", + "bg-rose-800", + "bg-rose-900", + "bg-rose-950", + "bg-origin-border", + "bg-origin-padding", + "bg-origin-content", + "bg-bottom", + "bg-center", + "bg-left", + "bg-left-bottom", + "bg-left-top", + "bg-right", + "bg-right-bottom", + "bg-right-top", + "bg-top", + "bg-repeat", + "bg-no-repeat", + "bg-repeat-x", + "bg-repeat-y", + "bg-repeat-round", + "bg-repeat-space", + "bg-auto", + "bg-cover", + "bg-contain", + "bg-none", + "bg-gradient-to-t", + "bg-gradient-to-tr", + "bg-gradient-to-r", + "bg-gradient-to-br", + "bg-gradient-to-b", + "bg-gradient-to-bl", + "bg-gradient-to-l", + "bg-gradient-to-tl", + "from-inherit", + "from-current", + "from-transparent", + "from-black", + "from-white", + "from-slate-50", + "from-slate-100", + "from-slate-200", + "from-slate-300", + "from-slate-400", + "from-slate-500", + "from-slate-600", + "from-slate-700", + "from-slate-800", + "from-slate-900", + "from-slate-950", + "from-gray-50", + "from-gray-100", + "from-gray-200", + "from-gray-300", + "from-gray-400", + "from-gray-500", + "from-gray-600", + "from-gray-700", + "from-gray-800", + "from-gray-900", + "from-gray-950", + "from-zinc-50", + "from-zinc-100", + "from-zinc-200", + "from-zinc-300", + "from-zinc-400", + "from-zinc-500", + "from-zinc-600", + "from-zinc-700", + "from-zinc-800", + "from-zinc-900", + "from-zinc-950", + "from-neutral-50", + "from-neutral-100", + "from-neutral-200", + "from-neutral-300", + "from-neutral-400", + "from-neutral-500", + "from-neutral-600", + "from-neutral-700", + "from-neutral-800", + "from-neutral-900", + "from-neutral-950", + "from-stone-50", + "from-stone-100", + "from-stone-200", + "from-stone-300", + "from-stone-400", + "from-stone-500", + "from-stone-600", + "from-stone-700", + "from-stone-800", + "from-stone-900", + "from-stone-950", + "from-red-50", + "from-red-100", + "from-red-200", + "from-red-300", + "from-red-400", + "from-red-500", + "from-red-600", + "from-red-700", + "from-red-800", + "from-red-900", + "from-red-950", + "from-orange-50", + "from-orange-100", + "from-orange-200", + "from-orange-300", + "from-orange-400", + "from-orange-500", + "from-orange-600", + "from-orange-700", + "from-orange-800", + "from-orange-900", + "from-orange-950", + "from-amber-50", + "from-amber-100", + "from-amber-200", + "from-amber-300", + "from-amber-400", + "from-amber-500", + "from-amber-600", + "from-amber-700", + "from-amber-800", + "from-amber-900", + "from-amber-950", + "from-yellow-50", + "from-yellow-100", + "from-yellow-200", + "from-yellow-300", + "from-yellow-400", + "from-yellow-500", + "from-yellow-600", + "from-yellow-700", + "from-yellow-800", + "from-yellow-900", + "from-yellow-950", + "from-lime-50", + "from-lime-100", + "from-lime-200", + "from-lime-300", + "from-lime-400", + "from-lime-500", + "from-lime-600", + "from-lime-700", + "from-lime-800", + "from-lime-900", + "from-lime-950", + "from-green-50", + "from-green-100", + "from-green-200", + "from-green-300", + "from-green-400", + "from-green-500", + "from-green-600", + "from-green-700", + "from-green-800", + "from-green-900", + "from-green-950", + "from-emerald-50", + "from-emerald-100", + "from-emerald-200", + "from-emerald-300", + "from-emerald-400", + "from-emerald-500", + "from-emerald-600", + "from-emerald-700", + "from-emerald-800", + "from-emerald-900", + "from-emerald-950", + "from-teal-50", + "from-teal-100", + "from-teal-200", + "from-teal-300", + "from-teal-400", + "from-teal-500", + "from-teal-600", + "from-teal-700", + "from-teal-800", + "from-teal-900", + "from-teal-950", + "from-cyan-50", + "from-cyan-100", + "from-cyan-200", + "from-cyan-300", + "from-cyan-400", + "from-cyan-500", + "from-cyan-600", + "from-cyan-700", + "from-cyan-800", + "from-cyan-900", + "from-cyan-950", + "from-sky-50", + "from-sky-100", + "from-sky-200", + "from-sky-300", + "from-sky-400", + "from-sky-500", + "from-sky-600", + "from-sky-700", + "from-sky-800", + "from-sky-900", + "from-sky-950", + "from-blue-50", + "from-blue-100", + "from-blue-200", + "from-blue-300", + "from-blue-400", + "from-blue-500", + "from-blue-600", + "from-blue-700", + "from-blue-800", + "from-blue-900", + "from-blue-950", + "from-indigo-50", + "from-indigo-100", + "from-indigo-200", + "from-indigo-300", + "from-indigo-400", + "from-indigo-500", + "from-indigo-600", + "from-indigo-700", + "from-indigo-800", + "from-indigo-900", + "from-indigo-950", + "from-violet-50", + "from-violet-100", + "from-violet-200", + "from-violet-300", + "from-violet-400", + "from-violet-500", + "from-violet-600", + "from-violet-700", + "from-violet-800", + "from-violet-900", + "from-violet-950", + "from-purple-50", + "from-purple-100", + "from-purple-200", + "from-purple-300", + "from-purple-400", + "from-purple-500", + "from-purple-600", + "from-purple-700", + "from-purple-800", + "from-purple-900", + "from-purple-950", + "from-fuchsia-50", + "from-fuchsia-100", + "from-fuchsia-200", + "from-fuchsia-300", + "from-fuchsia-400", + "from-fuchsia-500", + "from-fuchsia-600", + "from-fuchsia-700", + "from-fuchsia-800", + "from-fuchsia-900", + "from-fuchsia-950", + "from-pink-50", + "from-pink-100", + "from-pink-200", + "from-pink-300", + "from-pink-400", + "from-pink-500", + "from-pink-600", + "from-pink-700", + "from-pink-800", + "from-pink-900", + "from-pink-950", + "from-rose-50", + "from-rose-100", + "from-rose-200", + "from-rose-300", + "from-rose-400", + "from-rose-500", + "from-rose-600", + "from-rose-700", + "from-rose-800", + "from-rose-900", + "from-rose-950", + "from-0%", + "from-5%", + "from-10%", + "from-15%", + "from-20%", + "from-25%", + "from-30%", + "from-35%", + "from-40%", + "from-45%", + "from-50%", + "from-55%", + "from-60%", + "from-65%", + "from-70%", + "from-75%", + "from-80%", + "from-85%", + "from-90%", + "from-95%", + "from-100%", + "via-inherit", + "via-current", + "via-transparent", + "via-black", + "via-white", + "via-slate-50", + "via-slate-100", + "via-slate-200", + "via-slate-300", + "via-slate-400", + "via-slate-500", + "via-slate-600", + "via-slate-700", + "via-slate-800", + "via-slate-900", + "via-slate-950", + "via-gray-50", + "via-gray-100", + "via-gray-200", + "via-gray-300", + "via-gray-400", + "via-gray-500", + "via-gray-600", + "via-gray-700", + "via-gray-800", + "via-gray-900", + "via-gray-950", + "via-zinc-50", + "via-zinc-100", + "via-zinc-200", + "via-zinc-300", + "via-zinc-400", + "via-zinc-500", + "via-zinc-600", + "via-zinc-700", + "via-zinc-800", + "via-zinc-900", + "via-zinc-950", + "via-neutral-50", + "via-neutral-100", + "via-neutral-200", + "via-neutral-300", + "via-neutral-400", + "via-neutral-500", + "via-neutral-600", + "via-neutral-700", + "via-neutral-800", + "via-neutral-900", + "via-neutral-950", + "via-stone-50", + "via-stone-100", + "via-stone-200", + "via-stone-300", + "via-stone-400", + "via-stone-500", + "via-stone-600", + "via-stone-700", + "via-stone-800", + "via-stone-900", + "via-stone-950", + "via-red-50", + "via-red-100", + "via-red-200", + "via-red-300", + "via-red-400", + "via-red-500", + "via-red-600", + "via-red-700", + "via-red-800", + "via-red-900", + "via-red-950", + "via-orange-50", + "via-orange-100", + "via-orange-200", + "via-orange-300", + "via-orange-400", + "via-orange-500", + "via-orange-600", + "via-orange-700", + "via-orange-800", + "via-orange-900", + "via-orange-950", + "via-amber-50", + "via-amber-100", + "via-amber-200", + "via-amber-300", + "via-amber-400", + "via-amber-500", + "via-amber-600", + "via-amber-700", + "via-amber-800", + "via-amber-900", + "via-amber-950", + "via-yellow-50", + "via-yellow-100", + "via-yellow-200", + "via-yellow-300", + "via-yellow-400", + "via-yellow-500", + "via-yellow-600", + "via-yellow-700", + "via-yellow-800", + "via-yellow-900", + "via-yellow-950", + "via-lime-50", + "via-lime-100", + "via-lime-200", + "via-lime-300", + "via-lime-400", + "via-lime-500", + "via-lime-600", + "via-lime-700", + "via-lime-800", + "via-lime-900", + "via-lime-950", + "via-green-50", + "via-green-100", + "via-green-200", + "via-green-300", + "via-green-400", + "via-green-500", + "via-green-600", + "via-green-700", + "via-green-800", + "via-green-900", + "via-green-950", + "via-emerald-50", + "via-emerald-100", + "via-emerald-200", + "via-emerald-300", + "via-emerald-400", + "via-emerald-500", + "via-emerald-600", + "via-emerald-700", + "via-emerald-800", + "via-emerald-900", + "via-emerald-950", + "via-teal-50", + "via-teal-100", + "via-teal-200", + "via-teal-300", + "via-teal-400", + "via-teal-500", + "via-teal-600", + "via-teal-700", + "via-teal-800", + "via-teal-900", + "via-teal-950", + "via-cyan-50", + "via-cyan-100", + "via-cyan-200", + "via-cyan-300", + "via-cyan-400", + "via-cyan-500", + "via-cyan-600", + "via-cyan-700", + "via-cyan-800", + "via-cyan-900", + "via-cyan-950", + "via-sky-50", + "via-sky-100", + "via-sky-200", + "via-sky-300", + "via-sky-400", + "via-sky-500", + "via-sky-600", + "via-sky-700", + "via-sky-800", + "via-sky-900", + "via-sky-950", + "via-blue-50", + "via-blue-100", + "via-blue-200", + "via-blue-300", + "via-blue-400", + "via-blue-500", + "via-blue-600", + "via-blue-700", + "via-blue-800", + "via-blue-900", + "via-blue-950", + "via-indigo-50", + "via-indigo-100", + "via-indigo-200", + "via-indigo-300", + "via-indigo-400", + "via-indigo-500", + "via-indigo-600", + "via-indigo-700", + "via-indigo-800", + "via-indigo-900", + "via-indigo-950", + "via-violet-50", + "via-violet-100", + "via-violet-200", + "via-violet-300", + "via-violet-400", + "via-violet-500", + "via-violet-600", + "via-violet-700", + "via-violet-800", + "via-violet-900", + "via-violet-950", + "via-purple-50", + "via-purple-100", + "via-purple-200", + "via-purple-300", + "via-purple-400", + "via-purple-500", + "via-purple-600", + "via-purple-700", + "via-purple-800", + "via-purple-900", + "via-purple-950", + "via-fuchsia-50", + "via-fuchsia-100", + "via-fuchsia-200", + "via-fuchsia-300", + "via-fuchsia-400", + "via-fuchsia-500", + "via-fuchsia-600", + "via-fuchsia-700", + "via-fuchsia-800", + "via-fuchsia-900", + "via-fuchsia-950", + "via-pink-50", + "via-pink-100", + "via-pink-200", + "via-pink-300", + "via-pink-400", + "via-pink-500", + "via-pink-600", + "via-pink-700", + "via-pink-800", + "via-pink-900", + "via-pink-950", + "via-rose-50", + "via-rose-100", + "via-rose-200", + "via-rose-300", + "via-rose-400", + "via-rose-500", + "via-rose-600", + "via-rose-700", + "via-rose-800", + "via-rose-900", + "via-rose-950", + "via-0%", + "via-5%", + "via-10%", + "via-15%", + "via-20%", + "via-25%", + "via-30%", + "via-35%", + "via-40%", + "via-45%", + "via-50%", + "via-55%", + "via-60%", + "via-65%", + "via-70%", + "via-75%", + "via-80%", + "via-85%", + "via-90%", + "via-95%", + "via-100%", + "to-inherit", + "to-current", + "to-transparent", + "to-black", + "to-white", + "to-slate-50", + "to-slate-100", + "to-slate-200", + "to-slate-300", + "to-slate-400", + "to-slate-500", + "to-slate-600", + "to-slate-700", + "to-slate-800", + "to-slate-900", + "to-slate-950", + "to-gray-50", + "to-gray-100", + "to-gray-200", + "to-gray-300", + "to-gray-400", + "to-gray-500", + "to-gray-600", + "to-gray-700", + "to-gray-800", + "to-gray-900", + "to-gray-950", + "to-zinc-50", + "to-zinc-100", + "to-zinc-200", + "to-zinc-300", + "to-zinc-400", + "to-zinc-500", + "to-zinc-600", + "to-zinc-700", + "to-zinc-800", + "to-zinc-900", + "to-zinc-950", + "to-neutral-50", + "to-neutral-100", + "to-neutral-200", + "to-neutral-300", + "to-neutral-400", + "to-neutral-500", + "to-neutral-600", + "to-neutral-700", + "to-neutral-800", + "to-neutral-900", + "to-neutral-950", + "to-stone-50", + "to-stone-100", + "to-stone-200", + "to-stone-300", + "to-stone-400", + "to-stone-500", + "to-stone-600", + "to-stone-700", + "to-stone-800", + "to-stone-900", + "to-stone-950", + "to-red-50", + "to-red-100", + "to-red-200", + "to-red-300", + "to-red-400", + "to-red-500", + "to-red-600", + "to-red-700", + "to-red-800", + "to-red-900", + "to-red-950", + "to-orange-50", + "to-orange-100", + "to-orange-200", + "to-orange-300", + "to-orange-400", + "to-orange-500", + "to-orange-600", + "to-orange-700", + "to-orange-800", + "to-orange-900", + "to-orange-950", + "to-amber-50", + "to-amber-100", + "to-amber-200", + "to-amber-300", + "to-amber-400", + "to-amber-500", + "to-amber-600", + "to-amber-700", + "to-amber-800", + "to-amber-900", + "to-amber-950", + "to-yellow-50", + "to-yellow-100", + "to-yellow-200", + "to-yellow-300", + "to-yellow-400", + "to-yellow-500", + "to-yellow-600", + "to-yellow-700", + "to-yellow-800", + "to-yellow-900", + "to-yellow-950", + "to-lime-50", + "to-lime-100", + "to-lime-200", + "to-lime-300", + "to-lime-400", + "to-lime-500", + "to-lime-600", + "to-lime-700", + "to-lime-800", + "to-lime-900", + "to-lime-950", + "to-green-50", + "to-green-100", + "to-green-200", + "to-green-300", + "to-green-400", + "to-green-500", + "to-green-600", + "to-green-700", + "to-green-800", + "to-green-900", + "to-green-950", + "to-emerald-50", + "to-emerald-100", + "to-emerald-200", + "to-emerald-300", + "to-emerald-400", + "to-emerald-500", + "to-emerald-600", + "to-emerald-700", + "to-emerald-800", + "to-emerald-900", + "to-emerald-950", + "to-teal-50", + "to-teal-100", + "to-teal-200", + "to-teal-300", + "to-teal-400", + "to-teal-500", + "to-teal-600", + "to-teal-700", + "to-teal-800", + "to-teal-900", + "to-teal-950", + "to-cyan-50", + "to-cyan-100", + "to-cyan-200", + "to-cyan-300", + "to-cyan-400", + "to-cyan-500", + "to-cyan-600", + "to-cyan-700", + "to-cyan-800", + "to-cyan-900", + "to-cyan-950", + "to-sky-50", + "to-sky-100", + "to-sky-200", + "to-sky-300", + "to-sky-400", + "to-sky-500", + "to-sky-600", + "to-sky-700", + "to-sky-800", + "to-sky-900", + "to-sky-950", + "to-blue-50", + "to-blue-100", + "to-blue-200", + "to-blue-300", + "to-blue-400", + "to-blue-500", + "to-blue-600", + "to-blue-700", + "to-blue-800", + "to-blue-900", + "to-blue-950", + "to-indigo-50", + "to-indigo-100", + "to-indigo-200", + "to-indigo-300", + "to-indigo-400", + "to-indigo-500", + "to-indigo-600", + "to-indigo-700", + "to-indigo-800", + "to-indigo-900", + "to-indigo-950", + "to-violet-50", + "to-violet-100", + "to-violet-200", + "to-violet-300", + "to-violet-400", + "to-violet-500", + "to-violet-600", + "to-violet-700", + "to-violet-800", + "to-violet-900", + "to-violet-950", + "to-purple-50", + "to-purple-100", + "to-purple-200", + "to-purple-300", + "to-purple-400", + "to-purple-500", + "to-purple-600", + "to-purple-700", + "to-purple-800", + "to-purple-900", + "to-purple-950", + "to-fuchsia-50", + "to-fuchsia-100", + "to-fuchsia-200", + "to-fuchsia-300", + "to-fuchsia-400", + "to-fuchsia-500", + "to-fuchsia-600", + "to-fuchsia-700", + "to-fuchsia-800", + "to-fuchsia-900", + "to-fuchsia-950", + "to-pink-50", + "to-pink-100", + "to-pink-200", + "to-pink-300", + "to-pink-400", + "to-pink-500", + "to-pink-600", + "to-pink-700", + "to-pink-800", + "to-pink-900", + "to-pink-950", + "to-rose-50", + "to-rose-100", + "to-rose-200", + "to-rose-300", + "to-rose-400", + "to-rose-500", + "to-rose-600", + "to-rose-700", + "to-rose-800", + "to-rose-900", + "to-rose-950", + "to-0%", + "to-5%", + "to-10%", + "to-15%", + "to-20%", + "to-25%", + "to-30%", + "to-35%", + "to-40%", + "to-45%", + "to-50%", + "to-55%", + "to-60%", + "to-65%", + "to-70%", + "to-75%", + "to-80%", + "to-85%", + "to-90%", + "to-95%", + "to-100%", + "rounded-none", + "rounded-sm", + "rounded", + "rounded-md", + "rounded-lg", + "rounded-xl", + "rounded-2xl", + "rounded-3xl", + "rounded-full", + "rounded-s-none", + "rounded-s-sm", + "rounded-s", + "rounded-s-md", + "rounded-s-lg", + "rounded-s-xl", + "rounded-s-2xl", + "rounded-s-3xl", + "rounded-s-full", + "rounded-e-none", + "rounded-e-sm", + "rounded-e", + "rounded-e-md", + "rounded-e-lg", + "rounded-e-xl", + "rounded-e-2xl", + "rounded-e-3xl", + "rounded-e-full", + "rounded-t-none", + "rounded-t-sm", + "rounded-t", + "rounded-t-md", + "rounded-t-lg", + "rounded-t-xl", + "rounded-t-2xl", + "rounded-t-3xl", + "rounded-t-full", + "rounded-r-none", + "rounded-r-sm", + "rounded-r", + "rounded-r-md", + "rounded-r-lg", + "rounded-r-xl", + "rounded-r-2xl", + "rounded-r-3xl", + "rounded-r-full", + "rounded-b-none", + "rounded-b-sm", + "rounded-b", + "rounded-b-md", + "rounded-b-lg", + "rounded-b-xl", + "rounded-b-2xl", + "rounded-b-3xl", + "rounded-b-full", + "rounded-l-none", + "rounded-l-sm", + "rounded-l", + "rounded-l-md", + "rounded-l-lg", + "rounded-l-xl", + "rounded-l-2xl", + "rounded-l-3xl", + "rounded-l-full", + "rounded-ss-none", + "rounded-ss-sm", + "rounded-ss", + "rounded-ss-md", + "rounded-ss-lg", + "rounded-ss-xl", + "rounded-ss-2xl", + "rounded-ss-3xl", + "rounded-ss-full", + "rounded-se-none", + "rounded-se-sm", + "rounded-se", + "rounded-se-md", + "rounded-se-lg", + "rounded-se-xl", + "rounded-se-2xl", + "rounded-se-3xl", + "rounded-se-full", + "rounded-ee-none", + "rounded-ee-sm", + "rounded-ee", + "rounded-ee-md", + "rounded-ee-lg", + "rounded-ee-xl", + "rounded-ee-2xl", + "rounded-ee-3xl", + "rounded-ee-full", + "rounded-es-none", + "rounded-es-sm", + "rounded-es", + "rounded-es-md", + "rounded-es-lg", + "rounded-es-xl", + "rounded-es-2xl", + "rounded-es-3xl", + "rounded-es-full", + "rounded-tl-none", + "rounded-tl-sm", + "rounded-tl", + "rounded-tl-md", + "rounded-tl-lg", + "rounded-tl-xl", + "rounded-tl-2xl", + "rounded-tl-3xl", + "rounded-tl-full", + "rounded-tr-none", + "rounded-tr-sm", + "rounded-tr", + "rounded-tr-md", + "rounded-tr-lg", + "rounded-tr-xl", + "rounded-tr-2xl", + "rounded-tr-3xl", + "rounded-tr-full", + "rounded-br-none", + "rounded-br-sm", + "rounded-br", + "rounded-br-md", + "rounded-br-lg", + "rounded-br-xl", + "rounded-br-2xl", + "rounded-br-3xl", + "rounded-br-full", + "rounded-bl-none", + "rounded-bl-sm", + "rounded-bl", + "rounded-bl-md", + "rounded-bl-lg", + "rounded-bl-xl", + "rounded-bl-2xl", + "rounded-bl-3xl", + "rounded-bl-full", + "rounded-s-*", + "rounded-e-*", + "rounded-ss-*", + "rounded-se-*", + "rounded-es-*", + "rounded-ee-*", + "border-0", + "border-2", + "border-4", + "border-8", + "border", + "border-x-0", + "border-x-2", + "border-x-4", + "border-x-8", + "border-x", + "border-y-0", + "border-y-2", + "border-y-4", + "border-y-8", + "border-y", + "border-s-0", + "border-s-2", + "border-s-4", + "border-s-8", + "border-s", + "border-e-0", + "border-e-2", + "border-e-4", + "border-e-8", + "border-e", + "border-t-0", + "border-t-2", + "border-t-4", + "border-t-8", + "border-t", + "border-r-0", + "border-r-2", + "border-r-4", + "border-r-8", + "border-r", + "border-b-0", + "border-b-2", + "border-b-4", + "border-b-8", + "border-b", + "border-l-0", + "border-l-2", + "border-l-4", + "border-l-8", + "border-l", + "border-inherit", + "border-current", + "border-transparent", + "border-black", + "border-white", + "border-slate-50", + "border-slate-100", + "border-slate-200", + "border-slate-300", + "border-slate-400", + "border-slate-500", + "border-slate-600", + "border-slate-700", + "border-slate-800", + "border-slate-900", + "border-slate-950", + "border-gray-50", + "border-gray-100", + "border-gray-200", + "border-gray-300", + "border-gray-400", + "border-gray-500", + "border-gray-600", + "border-gray-700", + "border-gray-800", + "border-gray-900", + "border-gray-950", + "border-zinc-50", + "border-zinc-100", + "border-zinc-200", + "border-zinc-300", + "border-zinc-400", + "border-zinc-500", + "border-zinc-600", + "border-zinc-700", + "border-zinc-800", + "border-zinc-900", + "border-zinc-950", + "border-neutral-50", + "border-neutral-100", + "border-neutral-200", + "border-neutral-300", + "border-neutral-400", + "border-neutral-500", + "border-neutral-600", + "border-neutral-700", + "border-neutral-800", + "border-neutral-900", + "border-neutral-950", + "border-stone-50", + "border-stone-100", + "border-stone-200", + "border-stone-300", + "border-stone-400", + "border-stone-500", + "border-stone-600", + "border-stone-700", + "border-stone-800", + "border-stone-900", + "border-stone-950", + "border-red-50", + "border-red-100", + "border-red-200", + "border-red-300", + "border-red-400", + "border-red-500", + "border-red-600", + "border-red-700", + "border-red-800", + "border-red-900", + "border-red-950", + "border-orange-50", + "border-orange-100", + "border-orange-200", + "border-orange-300", + "border-orange-400", + "border-orange-500", + "border-orange-600", + "border-orange-700", + "border-orange-800", + "border-orange-900", + "border-orange-950", + "border-amber-50", + "border-amber-100", + "border-amber-200", + "border-amber-300", + "border-amber-400", + "border-amber-500", + "border-amber-600", + "border-amber-700", + "border-amber-800", + "border-amber-900", + "border-amber-950", + "border-yellow-50", + "border-yellow-100", + "border-yellow-200", + "border-yellow-300", + "border-yellow-400", + "border-yellow-500", + "border-yellow-600", + "border-yellow-700", + "border-yellow-800", + "border-yellow-900", + "border-yellow-950", + "border-lime-50", + "border-lime-100", + "border-lime-200", + "border-lime-300", + "border-lime-400", + "border-lime-500", + "border-lime-600", + "border-lime-700", + "border-lime-800", + "border-lime-900", + "border-lime-950", + "border-green-50", + "border-green-100", + "border-green-200", + "border-green-300", + "border-green-400", + "border-green-500", + "border-green-600", + "border-green-700", + "border-green-800", + "border-green-900", + "border-green-950", + "border-emerald-50", + "border-emerald-100", + "border-emerald-200", + "border-emerald-300", + "border-emerald-400", + "border-emerald-500", + "border-emerald-600", + "border-emerald-700", + "border-emerald-800", + "border-emerald-900", + "border-emerald-950", + "border-teal-50", + "border-teal-100", + "border-teal-200", + "border-teal-300", + "border-teal-400", + "border-teal-500", + "border-teal-600", + "border-teal-700", + "border-teal-800", + "border-teal-900", + "border-teal-950", + "border-cyan-50", + "border-cyan-100", + "border-cyan-200", + "border-cyan-300", + "border-cyan-400", + "border-cyan-500", + "border-cyan-600", + "border-cyan-700", + "border-cyan-800", + "border-cyan-900", + "border-cyan-950", + "border-sky-50", + "border-sky-100", + "border-sky-200", + "border-sky-300", + "border-sky-400", + "border-sky-500", + "border-sky-600", + "border-sky-700", + "border-sky-800", + "border-sky-900", + "border-sky-950", + "border-blue-50", + "border-blue-100", + "border-blue-200", + "border-blue-300", + "border-blue-400", + "border-blue-500", + "border-blue-600", + "border-blue-700", + "border-blue-800", + "border-blue-900", + "border-blue-950", + "border-indigo-50", + "border-indigo-100", + "border-indigo-200", + "border-indigo-300", + "border-indigo-400", + "border-indigo-500", + "border-indigo-600", + "border-indigo-700", + "border-indigo-800", + "border-indigo-900", + "border-indigo-950", + "border-violet-50", + "border-violet-100", + "border-violet-200", + "border-violet-300", + "border-violet-400", + "border-violet-500", + "border-violet-600", + "border-violet-700", + "border-violet-800", + "border-violet-900", + "border-violet-950", + "border-purple-50", + "border-purple-100", + "border-purple-200", + "border-purple-300", + "border-purple-400", + "border-purple-500", + "border-purple-600", + "border-purple-700", + "border-purple-800", + "border-purple-900", + "border-purple-950", + "border-fuchsia-50", + "border-fuchsia-100", + "border-fuchsia-200", + "border-fuchsia-300", + "border-fuchsia-400", + "border-fuchsia-500", + "border-fuchsia-600", + "border-fuchsia-700", + "border-fuchsia-800", + "border-fuchsia-900", + "border-fuchsia-950", + "border-pink-50", + "border-pink-100", + "border-pink-200", + "border-pink-300", + "border-pink-400", + "border-pink-500", + "border-pink-600", + "border-pink-700", + "border-pink-800", + "border-pink-900", + "border-pink-950", + "border-rose-50", + "border-rose-100", + "border-rose-200", + "border-rose-300", + "border-rose-400", + "border-rose-500", + "border-rose-600", + "border-rose-700", + "border-rose-800", + "border-rose-900", + "border-rose-950", + "border-x-inherit", + "border-x-current", + "border-x-transparent", + "border-x-black", + "border-x-white", + "border-x-slate-50", + "border-x-slate-100", + "border-x-slate-200", + "border-x-slate-300", + "border-x-slate-400", + "border-x-slate-500", + "border-x-slate-600", + "border-x-slate-700", + "border-x-slate-800", + "border-x-slate-900", + "border-x-slate-950", + "border-x-gray-50", + "border-x-gray-100", + "border-x-gray-200", + "border-x-gray-300", + "border-x-gray-400", + "border-x-gray-500", + "border-x-gray-600", + "border-x-gray-700", + "border-x-gray-800", + "border-x-gray-900", + "border-x-gray-950", + "border-x-zinc-50", + "border-x-zinc-100", + "border-x-zinc-200", + "border-x-zinc-300", + "border-x-zinc-400", + "border-x-zinc-500", + "border-x-zinc-600", + "border-x-zinc-700", + "border-x-zinc-800", + "border-x-zinc-900", + "border-x-zinc-950", + "border-x-neutral-50", + "border-x-neutral-100", + "border-x-neutral-200", + "border-x-neutral-300", + "border-x-neutral-400", + "border-x-neutral-500", + "border-x-neutral-600", + "border-x-neutral-700", + "border-x-neutral-800", + "border-x-neutral-900", + "border-x-neutral-950", + "border-x-stone-50", + "border-x-stone-100", + "border-x-stone-200", + "border-x-stone-300", + "border-x-stone-400", + "border-x-stone-500", + "border-x-stone-600", + "border-x-stone-700", + "border-x-stone-800", + "border-x-stone-900", + "border-x-stone-950", + "border-x-red-50", + "border-x-red-100", + "border-x-red-200", + "border-x-red-300", + "border-x-red-400", + "border-x-red-500", + "border-x-red-600", + "border-x-red-700", + "border-x-red-800", + "border-x-red-900", + "border-x-red-950", + "border-x-orange-50", + "border-x-orange-100", + "border-x-orange-200", + "border-x-orange-300", + "border-x-orange-400", + "border-x-orange-500", + "border-x-orange-600", + "border-x-orange-700", + "border-x-orange-800", + "border-x-orange-900", + "border-x-orange-950", + "border-x-amber-50", + "border-x-amber-100", + "border-x-amber-200", + "border-x-amber-300", + "border-x-amber-400", + "border-x-amber-500", + "border-x-amber-600", + "border-x-amber-700", + "border-x-amber-800", + "border-x-amber-900", + "border-x-amber-950", + "border-x-yellow-50", + "border-x-yellow-100", + "border-x-yellow-200", + "border-x-yellow-300", + "border-x-yellow-400", + "border-x-yellow-500", + "border-x-yellow-600", + "border-x-yellow-700", + "border-x-yellow-800", + "border-x-yellow-900", + "border-x-yellow-950", + "border-x-lime-50", + "border-x-lime-100", + "border-x-lime-200", + "border-x-lime-300", + "border-x-lime-400", + "border-x-lime-500", + "border-x-lime-600", + "border-x-lime-700", + "border-x-lime-800", + "border-x-lime-900", + "border-x-lime-950", + "border-x-green-50", + "border-x-green-100", + "border-x-green-200", + "border-x-green-300", + "border-x-green-400", + "border-x-green-500", + "border-x-green-600", + "border-x-green-700", + "border-x-green-800", + "border-x-green-900", + "border-x-green-950", + "border-x-emerald-50", + "border-x-emerald-100", + "border-x-emerald-200", + "border-x-emerald-300", + "border-x-emerald-400", + "border-x-emerald-500", + "border-x-emerald-600", + "border-x-emerald-700", + "border-x-emerald-800", + "border-x-emerald-900", + "border-x-emerald-950", + "border-x-teal-50", + "border-x-teal-100", + "border-x-teal-200", + "border-x-teal-300", + "border-x-teal-400", + "border-x-teal-500", + "border-x-teal-600", + "border-x-teal-700", + "border-x-teal-800", + "border-x-teal-900", + "border-x-teal-950", + "border-x-cyan-50", + "border-x-cyan-100", + "border-x-cyan-200", + "border-x-cyan-300", + "border-x-cyan-400", + "border-x-cyan-500", + "border-x-cyan-600", + "border-x-cyan-700", + "border-x-cyan-800", + "border-x-cyan-900", + "border-x-cyan-950", + "border-x-sky-50", + "border-x-sky-100", + "border-x-sky-200", + "border-x-sky-300", + "border-x-sky-400", + "border-x-sky-500", + "border-x-sky-600", + "border-x-sky-700", + "border-x-sky-800", + "border-x-sky-900", + "border-x-sky-950", + "border-x-blue-50", + "border-x-blue-100", + "border-x-blue-200", + "border-x-blue-300", + "border-x-blue-400", + "border-x-blue-500", + "border-x-blue-600", + "border-x-blue-700", + "border-x-blue-800", + "border-x-blue-900", + "border-x-blue-950", + "border-x-indigo-50", + "border-x-indigo-100", + "border-x-indigo-200", + "border-x-indigo-300", + "border-x-indigo-400", + "border-x-indigo-500", + "border-x-indigo-600", + "border-x-indigo-700", + "border-x-indigo-800", + "border-x-indigo-900", + "border-x-indigo-950", + "border-x-violet-50", + "border-x-violet-100", + "border-x-violet-200", + "border-x-violet-300", + "border-x-violet-400", + "border-x-violet-500", + "border-x-violet-600", + "border-x-violet-700", + "border-x-violet-800", + "border-x-violet-900", + "border-x-violet-950", + "border-x-purple-50", + "border-x-purple-100", + "border-x-purple-200", + "border-x-purple-300", + "border-x-purple-400", + "border-x-purple-500", + "border-x-purple-600", + "border-x-purple-700", + "border-x-purple-800", + "border-x-purple-900", + "border-x-purple-950", + "border-x-fuchsia-50", + "border-x-fuchsia-100", + "border-x-fuchsia-200", + "border-x-fuchsia-300", + "border-x-fuchsia-400", + "border-x-fuchsia-500", + "border-x-fuchsia-600", + "border-x-fuchsia-700", + "border-x-fuchsia-800", + "border-x-fuchsia-900", + "border-x-fuchsia-950", + "border-x-pink-50", + "border-x-pink-100", + "border-x-pink-200", + "border-x-pink-300", + "border-x-pink-400", + "border-x-pink-500", + "border-x-pink-600", + "border-x-pink-700", + "border-x-pink-800", + "border-x-pink-900", + "border-x-pink-950", + "border-x-rose-50", + "border-x-rose-100", + "border-x-rose-200", + "border-x-rose-300", + "border-x-rose-400", + "border-x-rose-500", + "border-x-rose-600", + "border-x-rose-700", + "border-x-rose-800", + "border-x-rose-900", + "border-x-rose-950", + "border-y-inherit", + "border-y-current", + "border-y-transparent", + "border-y-black", + "border-y-white", + "border-y-slate-50", + "border-y-slate-100", + "border-y-slate-200", + "border-y-slate-300", + "border-y-slate-400", + "border-y-slate-500", + "border-y-slate-600", + "border-y-slate-700", + "border-y-slate-800", + "border-y-slate-900", + "border-y-slate-950", + "border-y-gray-50", + "border-y-gray-100", + "border-y-gray-200", + "border-y-gray-300", + "border-y-gray-400", + "border-y-gray-500", + "border-y-gray-600", + "border-y-gray-700", + "border-y-gray-800", + "border-y-gray-900", + "border-y-gray-950", + "border-y-zinc-50", + "border-y-zinc-100", + "border-y-zinc-200", + "border-y-zinc-300", + "border-y-zinc-400", + "border-y-zinc-500", + "border-y-zinc-600", + "border-y-zinc-700", + "border-y-zinc-800", + "border-y-zinc-900", + "border-y-zinc-950", + "border-y-neutral-50", + "border-y-neutral-100", + "border-y-neutral-200", + "border-y-neutral-300", + "border-y-neutral-400", + "border-y-neutral-500", + "border-y-neutral-600", + "border-y-neutral-700", + "border-y-neutral-800", + "border-y-neutral-900", + "border-y-neutral-950", + "border-y-stone-50", + "border-y-stone-100", + "border-y-stone-200", + "border-y-stone-300", + "border-y-stone-400", + "border-y-stone-500", + "border-y-stone-600", + "border-y-stone-700", + "border-y-stone-800", + "border-y-stone-900", + "border-y-stone-950", + "border-y-red-50", + "border-y-red-100", + "border-y-red-200", + "border-y-red-300", + "border-y-red-400", + "border-y-red-500", + "border-y-red-600", + "border-y-red-700", + "border-y-red-800", + "border-y-red-900", + "border-y-red-950", + "border-y-orange-50", + "border-y-orange-100", + "border-y-orange-200", + "border-y-orange-300", + "border-y-orange-400", + "border-y-orange-500", + "border-y-orange-600", + "border-y-orange-700", + "border-y-orange-800", + "border-y-orange-900", + "border-y-orange-950", + "border-y-amber-50", + "border-y-amber-100", + "border-y-amber-200", + "border-y-amber-300", + "border-y-amber-400", + "border-y-amber-500", + "border-y-amber-600", + "border-y-amber-700", + "border-y-amber-800", + "border-y-amber-900", + "border-y-amber-950", + "border-y-yellow-50", + "border-y-yellow-100", + "border-y-yellow-200", + "border-y-yellow-300", + "border-y-yellow-400", + "border-y-yellow-500", + "border-y-yellow-600", + "border-y-yellow-700", + "border-y-yellow-800", + "border-y-yellow-900", + "border-y-yellow-950", + "border-y-lime-50", + "border-y-lime-100", + "border-y-lime-200", + "border-y-lime-300", + "border-y-lime-400", + "border-y-lime-500", + "border-y-lime-600", + "border-y-lime-700", + "border-y-lime-800", + "border-y-lime-900", + "border-y-lime-950", + "border-y-green-50", + "border-y-green-100", + "border-y-green-200", + "border-y-green-300", + "border-y-green-400", + "border-y-green-500", + "border-y-green-600", + "border-y-green-700", + "border-y-green-800", + "border-y-green-900", + "border-y-green-950", + "border-y-emerald-50", + "border-y-emerald-100", + "border-y-emerald-200", + "border-y-emerald-300", + "border-y-emerald-400", + "border-y-emerald-500", + "border-y-emerald-600", + "border-y-emerald-700", + "border-y-emerald-800", + "border-y-emerald-900", + "border-y-emerald-950", + "border-y-teal-50", + "border-y-teal-100", + "border-y-teal-200", + "border-y-teal-300", + "border-y-teal-400", + "border-y-teal-500", + "border-y-teal-600", + "border-y-teal-700", + "border-y-teal-800", + "border-y-teal-900", + "border-y-teal-950", + "border-y-cyan-50", + "border-y-cyan-100", + "border-y-cyan-200", + "border-y-cyan-300", + "border-y-cyan-400", + "border-y-cyan-500", + "border-y-cyan-600", + "border-y-cyan-700", + "border-y-cyan-800", + "border-y-cyan-900", + "border-y-cyan-950", + "border-y-sky-50", + "border-y-sky-100", + "border-y-sky-200", + "border-y-sky-300", + "border-y-sky-400", + "border-y-sky-500", + "border-y-sky-600", + "border-y-sky-700", + "border-y-sky-800", + "border-y-sky-900", + "border-y-sky-950", + "border-y-blue-50", + "border-y-blue-100", + "border-y-blue-200", + "border-y-blue-300", + "border-y-blue-400", + "border-y-blue-500", + "border-y-blue-600", + "border-y-blue-700", + "border-y-blue-800", + "border-y-blue-900", + "border-y-blue-950", + "border-y-indigo-50", + "border-y-indigo-100", + "border-y-indigo-200", + "border-y-indigo-300", + "border-y-indigo-400", + "border-y-indigo-500", + "border-y-indigo-600", + "border-y-indigo-700", + "border-y-indigo-800", + "border-y-indigo-900", + "border-y-indigo-950", + "border-y-violet-50", + "border-y-violet-100", + "border-y-violet-200", + "border-y-violet-300", + "border-y-violet-400", + "border-y-violet-500", + "border-y-violet-600", + "border-y-violet-700", + "border-y-violet-800", + "border-y-violet-900", + "border-y-violet-950", + "border-y-purple-50", + "border-y-purple-100", + "border-y-purple-200", + "border-y-purple-300", + "border-y-purple-400", + "border-y-purple-500", + "border-y-purple-600", + "border-y-purple-700", + "border-y-purple-800", + "border-y-purple-900", + "border-y-purple-950", + "border-y-fuchsia-50", + "border-y-fuchsia-100", + "border-y-fuchsia-200", + "border-y-fuchsia-300", + "border-y-fuchsia-400", + "border-y-fuchsia-500", + "border-y-fuchsia-600", + "border-y-fuchsia-700", + "border-y-fuchsia-800", + "border-y-fuchsia-900", + "border-y-fuchsia-950", + "border-y-pink-50", + "border-y-pink-100", + "border-y-pink-200", + "border-y-pink-300", + "border-y-pink-400", + "border-y-pink-500", + "border-y-pink-600", + "border-y-pink-700", + "border-y-pink-800", + "border-y-pink-900", + "border-y-pink-950", + "border-y-rose-50", + "border-y-rose-100", + "border-y-rose-200", + "border-y-rose-300", + "border-y-rose-400", + "border-y-rose-500", + "border-y-rose-600", + "border-y-rose-700", + "border-y-rose-800", + "border-y-rose-900", + "border-y-rose-950", + "border-s-inherit", + "border-s-current", + "border-s-transparent", + "border-s-black", + "border-s-white", + "border-s-slate-50", + "border-s-slate-100", + "border-s-slate-200", + "border-s-slate-300", + "border-s-slate-400", + "border-s-slate-500", + "border-s-slate-600", + "border-s-slate-700", + "border-s-slate-800", + "border-s-slate-900", + "border-s-slate-950", + "border-s-gray-50", + "border-s-gray-100", + "border-s-gray-200", + "border-s-gray-300", + "border-s-gray-400", + "border-s-gray-500", + "border-s-gray-600", + "border-s-gray-700", + "border-s-gray-800", + "border-s-gray-900", + "border-s-gray-950", + "border-s-zinc-50", + "border-s-zinc-100", + "border-s-zinc-200", + "border-s-zinc-300", + "border-s-zinc-400", + "border-s-zinc-500", + "border-s-zinc-600", + "border-s-zinc-700", + "border-s-zinc-800", + "border-s-zinc-900", + "border-s-zinc-950", + "border-s-neutral-50", + "border-s-neutral-100", + "border-s-neutral-200", + "border-s-neutral-300", + "border-s-neutral-400", + "border-s-neutral-500", + "border-s-neutral-600", + "border-s-neutral-700", + "border-s-neutral-800", + "border-s-neutral-900", + "border-s-neutral-950", + "border-s-stone-50", + "border-s-stone-100", + "border-s-stone-200", + "border-s-stone-300", + "border-s-stone-400", + "border-s-stone-500", + "border-s-stone-600", + "border-s-stone-700", + "border-s-stone-800", + "border-s-stone-900", + "border-s-stone-950", + "border-s-red-50", + "border-s-red-100", + "border-s-red-200", + "border-s-red-300", + "border-s-red-400", + "border-s-red-500", + "border-s-red-600", + "border-s-red-700", + "border-s-red-800", + "border-s-red-900", + "border-s-red-950", + "border-s-orange-50", + "border-s-orange-100", + "border-s-orange-200", + "border-s-orange-300", + "border-s-orange-400", + "border-s-orange-500", + "border-s-orange-600", + "border-s-orange-700", + "border-s-orange-800", + "border-s-orange-900", + "border-s-orange-950", + "border-s-amber-50", + "border-s-amber-100", + "border-s-amber-200", + "border-s-amber-300", + "border-s-amber-400", + "border-s-amber-500", + "border-s-amber-600", + "border-s-amber-700", + "border-s-amber-800", + "border-s-amber-900", + "border-s-amber-950", + "border-s-yellow-50", + "border-s-yellow-100", + "border-s-yellow-200", + "border-s-yellow-300", + "border-s-yellow-400", + "border-s-yellow-500", + "border-s-yellow-600", + "border-s-yellow-700", + "border-s-yellow-800", + "border-s-yellow-900", + "border-s-yellow-950", + "border-s-lime-50", + "border-s-lime-100", + "border-s-lime-200", + "border-s-lime-300", + "border-s-lime-400", + "border-s-lime-500", + "border-s-lime-600", + "border-s-lime-700", + "border-s-lime-800", + "border-s-lime-900", + "border-s-lime-950", + "border-s-green-50", + "border-s-green-100", + "border-s-green-200", + "border-s-green-300", + "border-s-green-400", + "border-s-green-500", + "border-s-green-600", + "border-s-green-700", + "border-s-green-800", + "border-s-green-900", + "border-s-green-950", + "border-s-emerald-50", + "border-s-emerald-100", + "border-s-emerald-200", + "border-s-emerald-300", + "border-s-emerald-400", + "border-s-emerald-500", + "border-s-emerald-600", + "border-s-emerald-700", + "border-s-emerald-800", + "border-s-emerald-900", + "border-s-emerald-950", + "border-s-teal-50", + "border-s-teal-100", + "border-s-teal-200", + "border-s-teal-300", + "border-s-teal-400", + "border-s-teal-500", + "border-s-teal-600", + "border-s-teal-700", + "border-s-teal-800", + "border-s-teal-900", + "border-s-teal-950", + "border-s-cyan-50", + "border-s-cyan-100", + "border-s-cyan-200", + "border-s-cyan-300", + "border-s-cyan-400", + "border-s-cyan-500", + "border-s-cyan-600", + "border-s-cyan-700", + "border-s-cyan-800", + "border-s-cyan-900", + "border-s-cyan-950", + "border-s-sky-50", + "border-s-sky-100", + "border-s-sky-200", + "border-s-sky-300", + "border-s-sky-400", + "border-s-sky-500", + "border-s-sky-600", + "border-s-sky-700", + "border-s-sky-800", + "border-s-sky-900", + "border-s-sky-950", + "border-s-blue-50", + "border-s-blue-100", + "border-s-blue-200", + "border-s-blue-300", + "border-s-blue-400", + "border-s-blue-500", + "border-s-blue-600", + "border-s-blue-700", + "border-s-blue-800", + "border-s-blue-900", + "border-s-blue-950", + "border-s-indigo-50", + "border-s-indigo-100", + "border-s-indigo-200", + "border-s-indigo-300", + "border-s-indigo-400", + "border-s-indigo-500", + "border-s-indigo-600", + "border-s-indigo-700", + "border-s-indigo-800", + "border-s-indigo-900", + "border-s-indigo-950", + "border-s-violet-50", + "border-s-violet-100", + "border-s-violet-200", + "border-s-violet-300", + "border-s-violet-400", + "border-s-violet-500", + "border-s-violet-600", + "border-s-violet-700", + "border-s-violet-800", + "border-s-violet-900", + "border-s-violet-950", + "border-s-purple-50", + "border-s-purple-100", + "border-s-purple-200", + "border-s-purple-300", + "border-s-purple-400", + "border-s-purple-500", + "border-s-purple-600", + "border-s-purple-700", + "border-s-purple-800", + "border-s-purple-900", + "border-s-purple-950", + "border-s-fuchsia-50", + "border-s-fuchsia-100", + "border-s-fuchsia-200", + "border-s-fuchsia-300", + "border-s-fuchsia-400", + "border-s-fuchsia-500", + "border-s-fuchsia-600", + "border-s-fuchsia-700", + "border-s-fuchsia-800", + "border-s-fuchsia-900", + "border-s-fuchsia-950", + "border-s-pink-50", + "border-s-pink-100", + "border-s-pink-200", + "border-s-pink-300", + "border-s-pink-400", + "border-s-pink-500", + "border-s-pink-600", + "border-s-pink-700", + "border-s-pink-800", + "border-s-pink-900", + "border-s-pink-950", + "border-s-rose-50", + "border-s-rose-100", + "border-s-rose-200", + "border-s-rose-300", + "border-s-rose-400", + "border-s-rose-500", + "border-s-rose-600", + "border-s-rose-700", + "border-s-rose-800", + "border-s-rose-900", + "border-s-rose-950", + "border-e-inherit", + "border-e-current", + "border-e-transparent", + "border-e-black", + "border-e-white", + "border-e-slate-50", + "border-e-slate-100", + "border-e-slate-200", + "border-e-slate-300", + "border-e-slate-400", + "border-e-slate-500", + "border-e-slate-600", + "border-e-slate-700", + "border-e-slate-800", + "border-e-slate-900", + "border-e-slate-950", + "border-e-gray-50", + "border-e-gray-100", + "border-e-gray-200", + "border-e-gray-300", + "border-e-gray-400", + "border-e-gray-500", + "border-e-gray-600", + "border-e-gray-700", + "border-e-gray-800", + "border-e-gray-900", + "border-e-gray-950", + "border-e-zinc-50", + "border-e-zinc-100", + "border-e-zinc-200", + "border-e-zinc-300", + "border-e-zinc-400", + "border-e-zinc-500", + "border-e-zinc-600", + "border-e-zinc-700", + "border-e-zinc-800", + "border-e-zinc-900", + "border-e-zinc-950", + "border-e-neutral-50", + "border-e-neutral-100", + "border-e-neutral-200", + "border-e-neutral-300", + "border-e-neutral-400", + "border-e-neutral-500", + "border-e-neutral-600", + "border-e-neutral-700", + "border-e-neutral-800", + "border-e-neutral-900", + "border-e-neutral-950", + "border-e-stone-50", + "border-e-stone-100", + "border-e-stone-200", + "border-e-stone-300", + "border-e-stone-400", + "border-e-stone-500", + "border-e-stone-600", + "border-e-stone-700", + "border-e-stone-800", + "border-e-stone-900", + "border-e-stone-950", + "border-e-red-50", + "border-e-red-100", + "border-e-red-200", + "border-e-red-300", + "border-e-red-400", + "border-e-red-500", + "border-e-red-600", + "border-e-red-700", + "border-e-red-800", + "border-e-red-900", + "border-e-red-950", + "border-e-orange-50", + "border-e-orange-100", + "border-e-orange-200", + "border-e-orange-300", + "border-e-orange-400", + "border-e-orange-500", + "border-e-orange-600", + "border-e-orange-700", + "border-e-orange-800", + "border-e-orange-900", + "border-e-orange-950", + "border-e-amber-50", + "border-e-amber-100", + "border-e-amber-200", + "border-e-amber-300", + "border-e-amber-400", + "border-e-amber-500", + "border-e-amber-600", + "border-e-amber-700", + "border-e-amber-800", + "border-e-amber-900", + "border-e-amber-950", + "border-e-yellow-50", + "border-e-yellow-100", + "border-e-yellow-200", + "border-e-yellow-300", + "border-e-yellow-400", + "border-e-yellow-500", + "border-e-yellow-600", + "border-e-yellow-700", + "border-e-yellow-800", + "border-e-yellow-900", + "border-e-yellow-950", + "border-e-lime-50", + "border-e-lime-100", + "border-e-lime-200", + "border-e-lime-300", + "border-e-lime-400", + "border-e-lime-500", + "border-e-lime-600", + "border-e-lime-700", + "border-e-lime-800", + "border-e-lime-900", + "border-e-lime-950", + "border-e-green-50", + "border-e-green-100", + "border-e-green-200", + "border-e-green-300", + "border-e-green-400", + "border-e-green-500", + "border-e-green-600", + "border-e-green-700", + "border-e-green-800", + "border-e-green-900", + "border-e-green-950", + "border-e-emerald-50", + "border-e-emerald-100", + "border-e-emerald-200", + "border-e-emerald-300", + "border-e-emerald-400", + "border-e-emerald-500", + "border-e-emerald-600", + "border-e-emerald-700", + "border-e-emerald-800", + "border-e-emerald-900", + "border-e-emerald-950", + "border-e-teal-50", + "border-e-teal-100", + "border-e-teal-200", + "border-e-teal-300", + "border-e-teal-400", + "border-e-teal-500", + "border-e-teal-600", + "border-e-teal-700", + "border-e-teal-800", + "border-e-teal-900", + "border-e-teal-950", + "border-e-cyan-50", + "border-e-cyan-100", + "border-e-cyan-200", + "border-e-cyan-300", + "border-e-cyan-400", + "border-e-cyan-500", + "border-e-cyan-600", + "border-e-cyan-700", + "border-e-cyan-800", + "border-e-cyan-900", + "border-e-cyan-950", + "border-e-sky-50", + "border-e-sky-100", + "border-e-sky-200", + "border-e-sky-300", + "border-e-sky-400", + "border-e-sky-500", + "border-e-sky-600", + "border-e-sky-700", + "border-e-sky-800", + "border-e-sky-900", + "border-e-sky-950", + "border-e-blue-50", + "border-e-blue-100", + "border-e-blue-200", + "border-e-blue-300", + "border-e-blue-400", + "border-e-blue-500", + "border-e-blue-600", + "border-e-blue-700", + "border-e-blue-800", + "border-e-blue-900", + "border-e-blue-950", + "border-e-indigo-50", + "border-e-indigo-100", + "border-e-indigo-200", + "border-e-indigo-300", + "border-e-indigo-400", + "border-e-indigo-500", + "border-e-indigo-600", + "border-e-indigo-700", + "border-e-indigo-800", + "border-e-indigo-900", + "border-e-indigo-950", + "border-e-violet-50", + "border-e-violet-100", + "border-e-violet-200", + "border-e-violet-300", + "border-e-violet-400", + "border-e-violet-500", + "border-e-violet-600", + "border-e-violet-700", + "border-e-violet-800", + "border-e-violet-900", + "border-e-violet-950", + "border-e-purple-50", + "border-e-purple-100", + "border-e-purple-200", + "border-e-purple-300", + "border-e-purple-400", + "border-e-purple-500", + "border-e-purple-600", + "border-e-purple-700", + "border-e-purple-800", + "border-e-purple-900", + "border-e-purple-950", + "border-e-fuchsia-50", + "border-e-fuchsia-100", + "border-e-fuchsia-200", + "border-e-fuchsia-300", + "border-e-fuchsia-400", + "border-e-fuchsia-500", + "border-e-fuchsia-600", + "border-e-fuchsia-700", + "border-e-fuchsia-800", + "border-e-fuchsia-900", + "border-e-fuchsia-950", + "border-e-pink-50", + "border-e-pink-100", + "border-e-pink-200", + "border-e-pink-300", + "border-e-pink-400", + "border-e-pink-500", + "border-e-pink-600", + "border-e-pink-700", + "border-e-pink-800", + "border-e-pink-900", + "border-e-pink-950", + "border-e-rose-50", + "border-e-rose-100", + "border-e-rose-200", + "border-e-rose-300", + "border-e-rose-400", + "border-e-rose-500", + "border-e-rose-600", + "border-e-rose-700", + "border-e-rose-800", + "border-e-rose-900", + "border-e-rose-950", + "border-t-inherit", + "border-t-current", + "border-t-transparent", + "border-t-black", + "border-t-white", + "border-t-slate-50", + "border-t-slate-100", + "border-t-slate-200", + "border-t-slate-300", + "border-t-slate-400", + "border-t-slate-500", + "border-t-slate-600", + "border-t-slate-700", + "border-t-slate-800", + "border-t-slate-900", + "border-t-slate-950", + "border-t-gray-50", + "border-t-gray-100", + "border-t-gray-200", + "border-t-gray-300", + "border-t-gray-400", + "border-t-gray-500", + "border-t-gray-600", + "border-t-gray-700", + "border-t-gray-800", + "border-t-gray-900", + "border-t-gray-950", + "border-t-zinc-50", + "border-t-zinc-100", + "border-t-zinc-200", + "border-t-zinc-300", + "border-t-zinc-400", + "border-t-zinc-500", + "border-t-zinc-600", + "border-t-zinc-700", + "border-t-zinc-800", + "border-t-zinc-900", + "border-t-zinc-950", + "border-t-neutral-50", + "border-t-neutral-100", + "border-t-neutral-200", + "border-t-neutral-300", + "border-t-neutral-400", + "border-t-neutral-500", + "border-t-neutral-600", + "border-t-neutral-700", + "border-t-neutral-800", + "border-t-neutral-900", + "border-t-neutral-950", + "border-t-stone-50", + "border-t-stone-100", + "border-t-stone-200", + "border-t-stone-300", + "border-t-stone-400", + "border-t-stone-500", + "border-t-stone-600", + "border-t-stone-700", + "border-t-stone-800", + "border-t-stone-900", + "border-t-stone-950", + "border-t-red-50", + "border-t-red-100", + "border-t-red-200", + "border-t-red-300", + "border-t-red-400", + "border-t-red-500", + "border-t-red-600", + "border-t-red-700", + "border-t-red-800", + "border-t-red-900", + "border-t-red-950", + "border-t-orange-50", + "border-t-orange-100", + "border-t-orange-200", + "border-t-orange-300", + "border-t-orange-400", + "border-t-orange-500", + "border-t-orange-600", + "border-t-orange-700", + "border-t-orange-800", + "border-t-orange-900", + "border-t-orange-950", + "border-t-amber-50", + "border-t-amber-100", + "border-t-amber-200", + "border-t-amber-300", + "border-t-amber-400", + "border-t-amber-500", + "border-t-amber-600", + "border-t-amber-700", + "border-t-amber-800", + "border-t-amber-900", + "border-t-amber-950", + "border-t-yellow-50", + "border-t-yellow-100", + "border-t-yellow-200", + "border-t-yellow-300", + "border-t-yellow-400", + "border-t-yellow-500", + "border-t-yellow-600", + "border-t-yellow-700", + "border-t-yellow-800", + "border-t-yellow-900", + "border-t-yellow-950", + "border-t-lime-50", + "border-t-lime-100", + "border-t-lime-200", + "border-t-lime-300", + "border-t-lime-400", + "border-t-lime-500", + "border-t-lime-600", + "border-t-lime-700", + "border-t-lime-800", + "border-t-lime-900", + "border-t-lime-950", + "border-t-green-50", + "border-t-green-100", + "border-t-green-200", + "border-t-green-300", + "border-t-green-400", + "border-t-green-500", + "border-t-green-600", + "border-t-green-700", + "border-t-green-800", + "border-t-green-900", + "border-t-green-950", + "border-t-emerald-50", + "border-t-emerald-100", + "border-t-emerald-200", + "border-t-emerald-300", + "border-t-emerald-400", + "border-t-emerald-500", + "border-t-emerald-600", + "border-t-emerald-700", + "border-t-emerald-800", + "border-t-emerald-900", + "border-t-emerald-950", + "border-t-teal-50", + "border-t-teal-100", + "border-t-teal-200", + "border-t-teal-300", + "border-t-teal-400", + "border-t-teal-500", + "border-t-teal-600", + "border-t-teal-700", + "border-t-teal-800", + "border-t-teal-900", + "border-t-teal-950", + "border-t-cyan-50", + "border-t-cyan-100", + "border-t-cyan-200", + "border-t-cyan-300", + "border-t-cyan-400", + "border-t-cyan-500", + "border-t-cyan-600", + "border-t-cyan-700", + "border-t-cyan-800", + "border-t-cyan-900", + "border-t-cyan-950", + "border-t-sky-50", + "border-t-sky-100", + "border-t-sky-200", + "border-t-sky-300", + "border-t-sky-400", + "border-t-sky-500", + "border-t-sky-600", + "border-t-sky-700", + "border-t-sky-800", + "border-t-sky-900", + "border-t-sky-950", + "border-t-blue-50", + "border-t-blue-100", + "border-t-blue-200", + "border-t-blue-300", + "border-t-blue-400", + "border-t-blue-500", + "border-t-blue-600", + "border-t-blue-700", + "border-t-blue-800", + "border-t-blue-900", + "border-t-blue-950", + "border-t-indigo-50", + "border-t-indigo-100", + "border-t-indigo-200", + "border-t-indigo-300", + "border-t-indigo-400", + "border-t-indigo-500", + "border-t-indigo-600", + "border-t-indigo-700", + "border-t-indigo-800", + "border-t-indigo-900", + "border-t-indigo-950", + "border-t-violet-50", + "border-t-violet-100", + "border-t-violet-200", + "border-t-violet-300", + "border-t-violet-400", + "border-t-violet-500", + "border-t-violet-600", + "border-t-violet-700", + "border-t-violet-800", + "border-t-violet-900", + "border-t-violet-950", + "border-t-purple-50", + "border-t-purple-100", + "border-t-purple-200", + "border-t-purple-300", + "border-t-purple-400", + "border-t-purple-500", + "border-t-purple-600", + "border-t-purple-700", + "border-t-purple-800", + "border-t-purple-900", + "border-t-purple-950", + "border-t-fuchsia-50", + "border-t-fuchsia-100", + "border-t-fuchsia-200", + "border-t-fuchsia-300", + "border-t-fuchsia-400", + "border-t-fuchsia-500", + "border-t-fuchsia-600", + "border-t-fuchsia-700", + "border-t-fuchsia-800", + "border-t-fuchsia-900", + "border-t-fuchsia-950", + "border-t-pink-50", + "border-t-pink-100", + "border-t-pink-200", + "border-t-pink-300", + "border-t-pink-400", + "border-t-pink-500", + "border-t-pink-600", + "border-t-pink-700", + "border-t-pink-800", + "border-t-pink-900", + "border-t-pink-950", + "border-t-rose-50", + "border-t-rose-100", + "border-t-rose-200", + "border-t-rose-300", + "border-t-rose-400", + "border-t-rose-500", + "border-t-rose-600", + "border-t-rose-700", + "border-t-rose-800", + "border-t-rose-900", + "border-t-rose-950", + "border-r-inherit", + "border-r-current", + "border-r-transparent", + "border-r-black", + "border-r-white", + "border-r-slate-50", + "border-r-slate-100", + "border-r-slate-200", + "border-r-slate-300", + "border-r-slate-400", + "border-r-slate-500", + "border-r-slate-600", + "border-r-slate-700", + "border-r-slate-800", + "border-r-slate-900", + "border-r-slate-950", + "border-r-gray-50", + "border-r-gray-100", + "border-r-gray-200", + "border-r-gray-300", + "border-r-gray-400", + "border-r-gray-500", + "border-r-gray-600", + "border-r-gray-700", + "border-r-gray-800", + "border-r-gray-900", + "border-r-gray-950", + "border-r-zinc-50", + "border-r-zinc-100", + "border-r-zinc-200", + "border-r-zinc-300", + "border-r-zinc-400", + "border-r-zinc-500", + "border-r-zinc-600", + "border-r-zinc-700", + "border-r-zinc-800", + "border-r-zinc-900", + "border-r-zinc-950", + "border-r-neutral-50", + "border-r-neutral-100", + "border-r-neutral-200", + "border-r-neutral-300", + "border-r-neutral-400", + "border-r-neutral-500", + "border-r-neutral-600", + "border-r-neutral-700", + "border-r-neutral-800", + "border-r-neutral-900", + "border-r-neutral-950", + "border-r-stone-50", + "border-r-stone-100", + "border-r-stone-200", + "border-r-stone-300", + "border-r-stone-400", + "border-r-stone-500", + "border-r-stone-600", + "border-r-stone-700", + "border-r-stone-800", + "border-r-stone-900", + "border-r-stone-950", + "border-r-red-50", + "border-r-red-100", + "border-r-red-200", + "border-r-red-300", + "border-r-red-400", + "border-r-red-500", + "border-r-red-600", + "border-r-red-700", + "border-r-red-800", + "border-r-red-900", + "border-r-red-950", + "border-r-orange-50", + "border-r-orange-100", + "border-r-orange-200", + "border-r-orange-300", + "border-r-orange-400", + "border-r-orange-500", + "border-r-orange-600", + "border-r-orange-700", + "border-r-orange-800", + "border-r-orange-900", + "border-r-orange-950", + "border-r-amber-50", + "border-r-amber-100", + "border-r-amber-200", + "border-r-amber-300", + "border-r-amber-400", + "border-r-amber-500", + "border-r-amber-600", + "border-r-amber-700", + "border-r-amber-800", + "border-r-amber-900", + "border-r-amber-950", + "border-r-yellow-50", + "border-r-yellow-100", + "border-r-yellow-200", + "border-r-yellow-300", + "border-r-yellow-400", + "border-r-yellow-500", + "border-r-yellow-600", + "border-r-yellow-700", + "border-r-yellow-800", + "border-r-yellow-900", + "border-r-yellow-950", + "border-r-lime-50", + "border-r-lime-100", + "border-r-lime-200", + "border-r-lime-300", + "border-r-lime-400", + "border-r-lime-500", + "border-r-lime-600", + "border-r-lime-700", + "border-r-lime-800", + "border-r-lime-900", + "border-r-lime-950", + "border-r-green-50", + "border-r-green-100", + "border-r-green-200", + "border-r-green-300", + "border-r-green-400", + "border-r-green-500", + "border-r-green-600", + "border-r-green-700", + "border-r-green-800", + "border-r-green-900", + "border-r-green-950", + "border-r-emerald-50", + "border-r-emerald-100", + "border-r-emerald-200", + "border-r-emerald-300", + "border-r-emerald-400", + "border-r-emerald-500", + "border-r-emerald-600", + "border-r-emerald-700", + "border-r-emerald-800", + "border-r-emerald-900", + "border-r-emerald-950", + "border-r-teal-50", + "border-r-teal-100", + "border-r-teal-200", + "border-r-teal-300", + "border-r-teal-400", + "border-r-teal-500", + "border-r-teal-600", + "border-r-teal-700", + "border-r-teal-800", + "border-r-teal-900", + "border-r-teal-950", + "border-r-cyan-50", + "border-r-cyan-100", + "border-r-cyan-200", + "border-r-cyan-300", + "border-r-cyan-400", + "border-r-cyan-500", + "border-r-cyan-600", + "border-r-cyan-700", + "border-r-cyan-800", + "border-r-cyan-900", + "border-r-cyan-950", + "border-r-sky-50", + "border-r-sky-100", + "border-r-sky-200", + "border-r-sky-300", + "border-r-sky-400", + "border-r-sky-500", + "border-r-sky-600", + "border-r-sky-700", + "border-r-sky-800", + "border-r-sky-900", + "border-r-sky-950", + "border-r-blue-50", + "border-r-blue-100", + "border-r-blue-200", + "border-r-blue-300", + "border-r-blue-400", + "border-r-blue-500", + "border-r-blue-600", + "border-r-blue-700", + "border-r-blue-800", + "border-r-blue-900", + "border-r-blue-950", + "border-r-indigo-50", + "border-r-indigo-100", + "border-r-indigo-200", + "border-r-indigo-300", + "border-r-indigo-400", + "border-r-indigo-500", + "border-r-indigo-600", + "border-r-indigo-700", + "border-r-indigo-800", + "border-r-indigo-900", + "border-r-indigo-950", + "border-r-violet-50", + "border-r-violet-100", + "border-r-violet-200", + "border-r-violet-300", + "border-r-violet-400", + "border-r-violet-500", + "border-r-violet-600", + "border-r-violet-700", + "border-r-violet-800", + "border-r-violet-900", + "border-r-violet-950", + "border-r-purple-50", + "border-r-purple-100", + "border-r-purple-200", + "border-r-purple-300", + "border-r-purple-400", + "border-r-purple-500", + "border-r-purple-600", + "border-r-purple-700", + "border-r-purple-800", + "border-r-purple-900", + "border-r-purple-950", + "border-r-fuchsia-50", + "border-r-fuchsia-100", + "border-r-fuchsia-200", + "border-r-fuchsia-300", + "border-r-fuchsia-400", + "border-r-fuchsia-500", + "border-r-fuchsia-600", + "border-r-fuchsia-700", + "border-r-fuchsia-800", + "border-r-fuchsia-900", + "border-r-fuchsia-950", + "border-r-pink-50", + "border-r-pink-100", + "border-r-pink-200", + "border-r-pink-300", + "border-r-pink-400", + "border-r-pink-500", + "border-r-pink-600", + "border-r-pink-700", + "border-r-pink-800", + "border-r-pink-900", + "border-r-pink-950", + "border-r-rose-50", + "border-r-rose-100", + "border-r-rose-200", + "border-r-rose-300", + "border-r-rose-400", + "border-r-rose-500", + "border-r-rose-600", + "border-r-rose-700", + "border-r-rose-800", + "border-r-rose-900", + "border-r-rose-950", + "border-b-inherit", + "border-b-current", + "border-b-transparent", + "border-b-black", + "border-b-white", + "border-b-slate-50", + "border-b-slate-100", + "border-b-slate-200", + "border-b-slate-300", + "border-b-slate-400", + "border-b-slate-500", + "border-b-slate-600", + "border-b-slate-700", + "border-b-slate-800", + "border-b-slate-900", + "border-b-slate-950", + "border-b-gray-50", + "border-b-gray-100", + "border-b-gray-200", + "border-b-gray-300", + "border-b-gray-400", + "border-b-gray-500", + "border-b-gray-600", + "border-b-gray-700", + "border-b-gray-800", + "border-b-gray-900", + "border-b-gray-950", + "border-b-zinc-50", + "border-b-zinc-100", + "border-b-zinc-200", + "border-b-zinc-300", + "border-b-zinc-400", + "border-b-zinc-500", + "border-b-zinc-600", + "border-b-zinc-700", + "border-b-zinc-800", + "border-b-zinc-900", + "border-b-zinc-950", + "border-b-neutral-50", + "border-b-neutral-100", + "border-b-neutral-200", + "border-b-neutral-300", + "border-b-neutral-400", + "border-b-neutral-500", + "border-b-neutral-600", + "border-b-neutral-700", + "border-b-neutral-800", + "border-b-neutral-900", + "border-b-neutral-950", + "border-b-stone-50", + "border-b-stone-100", + "border-b-stone-200", + "border-b-stone-300", + "border-b-stone-400", + "border-b-stone-500", + "border-b-stone-600", + "border-b-stone-700", + "border-b-stone-800", + "border-b-stone-900", + "border-b-stone-950", + "border-b-red-50", + "border-b-red-100", + "border-b-red-200", + "border-b-red-300", + "border-b-red-400", + "border-b-red-500", + "border-b-red-600", + "border-b-red-700", + "border-b-red-800", + "border-b-red-900", + "border-b-red-950", + "border-b-orange-50", + "border-b-orange-100", + "border-b-orange-200", + "border-b-orange-300", + "border-b-orange-400", + "border-b-orange-500", + "border-b-orange-600", + "border-b-orange-700", + "border-b-orange-800", + "border-b-orange-900", + "border-b-orange-950", + "border-b-amber-50", + "border-b-amber-100", + "border-b-amber-200", + "border-b-amber-300", + "border-b-amber-400", + "border-b-amber-500", + "border-b-amber-600", + "border-b-amber-700", + "border-b-amber-800", + "border-b-amber-900", + "border-b-amber-950", + "border-b-yellow-50", + "border-b-yellow-100", + "border-b-yellow-200", + "border-b-yellow-300", + "border-b-yellow-400", + "border-b-yellow-500", + "border-b-yellow-600", + "border-b-yellow-700", + "border-b-yellow-800", + "border-b-yellow-900", + "border-b-yellow-950", + "border-b-lime-50", + "border-b-lime-100", + "border-b-lime-200", + "border-b-lime-300", + "border-b-lime-400", + "border-b-lime-500", + "border-b-lime-600", + "border-b-lime-700", + "border-b-lime-800", + "border-b-lime-900", + "border-b-lime-950", + "border-b-green-50", + "border-b-green-100", + "border-b-green-200", + "border-b-green-300", + "border-b-green-400", + "border-b-green-500", + "border-b-green-600", + "border-b-green-700", + "border-b-green-800", + "border-b-green-900", + "border-b-green-950", + "border-b-emerald-50", + "border-b-emerald-100", + "border-b-emerald-200", + "border-b-emerald-300", + "border-b-emerald-400", + "border-b-emerald-500", + "border-b-emerald-600", + "border-b-emerald-700", + "border-b-emerald-800", + "border-b-emerald-900", + "border-b-emerald-950", + "border-b-teal-50", + "border-b-teal-100", + "border-b-teal-200", + "border-b-teal-300", + "border-b-teal-400", + "border-b-teal-500", + "border-b-teal-600", + "border-b-teal-700", + "border-b-teal-800", + "border-b-teal-900", + "border-b-teal-950", + "border-b-cyan-50", + "border-b-cyan-100", + "border-b-cyan-200", + "border-b-cyan-300", + "border-b-cyan-400", + "border-b-cyan-500", + "border-b-cyan-600", + "border-b-cyan-700", + "border-b-cyan-800", + "border-b-cyan-900", + "border-b-cyan-950", + "border-b-sky-50", + "border-b-sky-100", + "border-b-sky-200", + "border-b-sky-300", + "border-b-sky-400", + "border-b-sky-500", + "border-b-sky-600", + "border-b-sky-700", + "border-b-sky-800", + "border-b-sky-900", + "border-b-sky-950", + "border-b-blue-50", + "border-b-blue-100", + "border-b-blue-200", + "border-b-blue-300", + "border-b-blue-400", + "border-b-blue-500", + "border-b-blue-600", + "border-b-blue-700", + "border-b-blue-800", + "border-b-blue-900", + "border-b-blue-950", + "border-b-indigo-50", + "border-b-indigo-100", + "border-b-indigo-200", + "border-b-indigo-300", + "border-b-indigo-400", + "border-b-indigo-500", + "border-b-indigo-600", + "border-b-indigo-700", + "border-b-indigo-800", + "border-b-indigo-900", + "border-b-indigo-950", + "border-b-violet-50", + "border-b-violet-100", + "border-b-violet-200", + "border-b-violet-300", + "border-b-violet-400", + "border-b-violet-500", + "border-b-violet-600", + "border-b-violet-700", + "border-b-violet-800", + "border-b-violet-900", + "border-b-violet-950", + "border-b-purple-50", + "border-b-purple-100", + "border-b-purple-200", + "border-b-purple-300", + "border-b-purple-400", + "border-b-purple-500", + "border-b-purple-600", + "border-b-purple-700", + "border-b-purple-800", + "border-b-purple-900", + "border-b-purple-950", + "border-b-fuchsia-50", + "border-b-fuchsia-100", + "border-b-fuchsia-200", + "border-b-fuchsia-300", + "border-b-fuchsia-400", + "border-b-fuchsia-500", + "border-b-fuchsia-600", + "border-b-fuchsia-700", + "border-b-fuchsia-800", + "border-b-fuchsia-900", + "border-b-fuchsia-950", + "border-b-pink-50", + "border-b-pink-100", + "border-b-pink-200", + "border-b-pink-300", + "border-b-pink-400", + "border-b-pink-500", + "border-b-pink-600", + "border-b-pink-700", + "border-b-pink-800", + "border-b-pink-900", + "border-b-pink-950", + "border-b-rose-50", + "border-b-rose-100", + "border-b-rose-200", + "border-b-rose-300", + "border-b-rose-400", + "border-b-rose-500", + "border-b-rose-600", + "border-b-rose-700", + "border-b-rose-800", + "border-b-rose-900", + "border-b-rose-950", + "border-l-inherit", + "border-l-current", + "border-l-transparent", + "border-l-black", + "border-l-white", + "border-l-slate-50", + "border-l-slate-100", + "border-l-slate-200", + "border-l-slate-300", + "border-l-slate-400", + "border-l-slate-500", + "border-l-slate-600", + "border-l-slate-700", + "border-l-slate-800", + "border-l-slate-900", + "border-l-slate-950", + "border-l-gray-50", + "border-l-gray-100", + "border-l-gray-200", + "border-l-gray-300", + "border-l-gray-400", + "border-l-gray-500", + "border-l-gray-600", + "border-l-gray-700", + "border-l-gray-800", + "border-l-gray-900", + "border-l-gray-950", + "border-l-zinc-50", + "border-l-zinc-100", + "border-l-zinc-200", + "border-l-zinc-300", + "border-l-zinc-400", + "border-l-zinc-500", + "border-l-zinc-600", + "border-l-zinc-700", + "border-l-zinc-800", + "border-l-zinc-900", + "border-l-zinc-950", + "border-l-neutral-50", + "border-l-neutral-100", + "border-l-neutral-200", + "border-l-neutral-300", + "border-l-neutral-400", + "border-l-neutral-500", + "border-l-neutral-600", + "border-l-neutral-700", + "border-l-neutral-800", + "border-l-neutral-900", + "border-l-neutral-950", + "border-l-stone-50", + "border-l-stone-100", + "border-l-stone-200", + "border-l-stone-300", + "border-l-stone-400", + "border-l-stone-500", + "border-l-stone-600", + "border-l-stone-700", + "border-l-stone-800", + "border-l-stone-900", + "border-l-stone-950", + "border-l-red-50", + "border-l-red-100", + "border-l-red-200", + "border-l-red-300", + "border-l-red-400", + "border-l-red-500", + "border-l-red-600", + "border-l-red-700", + "border-l-red-800", + "border-l-red-900", + "border-l-red-950", + "border-l-orange-50", + "border-l-orange-100", + "border-l-orange-200", + "border-l-orange-300", + "border-l-orange-400", + "border-l-orange-500", + "border-l-orange-600", + "border-l-orange-700", + "border-l-orange-800", + "border-l-orange-900", + "border-l-orange-950", + "border-l-amber-50", + "border-l-amber-100", + "border-l-amber-200", + "border-l-amber-300", + "border-l-amber-400", + "border-l-amber-500", + "border-l-amber-600", + "border-l-amber-700", + "border-l-amber-800", + "border-l-amber-900", + "border-l-amber-950", + "border-l-yellow-50", + "border-l-yellow-100", + "border-l-yellow-200", + "border-l-yellow-300", + "border-l-yellow-400", + "border-l-yellow-500", + "border-l-yellow-600", + "border-l-yellow-700", + "border-l-yellow-800", + "border-l-yellow-900", + "border-l-yellow-950", + "border-l-lime-50", + "border-l-lime-100", + "border-l-lime-200", + "border-l-lime-300", + "border-l-lime-400", + "border-l-lime-500", + "border-l-lime-600", + "border-l-lime-700", + "border-l-lime-800", + "border-l-lime-900", + "border-l-lime-950", + "border-l-green-50", + "border-l-green-100", + "border-l-green-200", + "border-l-green-300", + "border-l-green-400", + "border-l-green-500", + "border-l-green-600", + "border-l-green-700", + "border-l-green-800", + "border-l-green-900", + "border-l-green-950", + "border-l-emerald-50", + "border-l-emerald-100", + "border-l-emerald-200", + "border-l-emerald-300", + "border-l-emerald-400", + "border-l-emerald-500", + "border-l-emerald-600", + "border-l-emerald-700", + "border-l-emerald-800", + "border-l-emerald-900", + "border-l-emerald-950", + "border-l-teal-50", + "border-l-teal-100", + "border-l-teal-200", + "border-l-teal-300", + "border-l-teal-400", + "border-l-teal-500", + "border-l-teal-600", + "border-l-teal-700", + "border-l-teal-800", + "border-l-teal-900", + "border-l-teal-950", + "border-l-cyan-50", + "border-l-cyan-100", + "border-l-cyan-200", + "border-l-cyan-300", + "border-l-cyan-400", + "border-l-cyan-500", + "border-l-cyan-600", + "border-l-cyan-700", + "border-l-cyan-800", + "border-l-cyan-900", + "border-l-cyan-950", + "border-l-sky-50", + "border-l-sky-100", + "border-l-sky-200", + "border-l-sky-300", + "border-l-sky-400", + "border-l-sky-500", + "border-l-sky-600", + "border-l-sky-700", + "border-l-sky-800", + "border-l-sky-900", + "border-l-sky-950", + "border-l-blue-50", + "border-l-blue-100", + "border-l-blue-200", + "border-l-blue-300", + "border-l-blue-400", + "border-l-blue-500", + "border-l-blue-600", + "border-l-blue-700", + "border-l-blue-800", + "border-l-blue-900", + "border-l-blue-950", + "border-l-indigo-50", + "border-l-indigo-100", + "border-l-indigo-200", + "border-l-indigo-300", + "border-l-indigo-400", + "border-l-indigo-500", + "border-l-indigo-600", + "border-l-indigo-700", + "border-l-indigo-800", + "border-l-indigo-900", + "border-l-indigo-950", + "border-l-violet-50", + "border-l-violet-100", + "border-l-violet-200", + "border-l-violet-300", + "border-l-violet-400", + "border-l-violet-500", + "border-l-violet-600", + "border-l-violet-700", + "border-l-violet-800", + "border-l-violet-900", + "border-l-violet-950", + "border-l-purple-50", + "border-l-purple-100", + "border-l-purple-200", + "border-l-purple-300", + "border-l-purple-400", + "border-l-purple-500", + "border-l-purple-600", + "border-l-purple-700", + "border-l-purple-800", + "border-l-purple-900", + "border-l-purple-950", + "border-l-fuchsia-50", + "border-l-fuchsia-100", + "border-l-fuchsia-200", + "border-l-fuchsia-300", + "border-l-fuchsia-400", + "border-l-fuchsia-500", + "border-l-fuchsia-600", + "border-l-fuchsia-700", + "border-l-fuchsia-800", + "border-l-fuchsia-900", + "border-l-fuchsia-950", + "border-l-pink-50", + "border-l-pink-100", + "border-l-pink-200", + "border-l-pink-300", + "border-l-pink-400", + "border-l-pink-500", + "border-l-pink-600", + "border-l-pink-700", + "border-l-pink-800", + "border-l-pink-900", + "border-l-pink-950", + "border-l-rose-50", + "border-l-rose-100", + "border-l-rose-200", + "border-l-rose-300", + "border-l-rose-400", + "border-l-rose-500", + "border-l-rose-600", + "border-l-rose-700", + "border-l-rose-800", + "border-l-rose-900", + "border-l-rose-950", + "border-solid", + "border-dashed", + "border-dotted", + "border-double", + "border-hidden", + "border-none", + "divide-x-0", + "divide-x-2", + "divide-x-4", + "divide-x-8", + "divide-x", + "divide-y-0", + "divide-y-2", + "divide-y-4", + "divide-y-8", + "divide-y", + "divide-y-reverse", + "divide-x-reverse", + "divide-inherit", + "divide-current", + "divide-transparent", + "divide-black", + "divide-white", + "divide-slate-50", + "divide-slate-100", + "divide-slate-200", + "divide-slate-300", + "divide-slate-400", + "divide-slate-500", + "divide-slate-600", + "divide-slate-700", + "divide-slate-800", + "divide-slate-900", + "divide-slate-950", + "divide-gray-50", + "divide-gray-100", + "divide-gray-200", + "divide-gray-300", + "divide-gray-400", + "divide-gray-500", + "divide-gray-600", + "divide-gray-700", + "divide-gray-800", + "divide-gray-900", + "divide-gray-950", + "divide-zinc-50", + "divide-zinc-100", + "divide-zinc-200", + "divide-zinc-300", + "divide-zinc-400", + "divide-zinc-500", + "divide-zinc-600", + "divide-zinc-700", + "divide-zinc-800", + "divide-zinc-900", + "divide-zinc-950", + "divide-neutral-50", + "divide-neutral-100", + "divide-neutral-200", + "divide-neutral-300", + "divide-neutral-400", + "divide-neutral-500", + "divide-neutral-600", + "divide-neutral-700", + "divide-neutral-800", + "divide-neutral-900", + "divide-neutral-950", + "divide-stone-50", + "divide-stone-100", + "divide-stone-200", + "divide-stone-300", + "divide-stone-400", + "divide-stone-500", + "divide-stone-600", + "divide-stone-700", + "divide-stone-800", + "divide-stone-900", + "divide-stone-950", + "divide-red-50", + "divide-red-100", + "divide-red-200", + "divide-red-300", + "divide-red-400", + "divide-red-500", + "divide-red-600", + "divide-red-700", + "divide-red-800", + "divide-red-900", + "divide-red-950", + "divide-orange-50", + "divide-orange-100", + "divide-orange-200", + "divide-orange-300", + "divide-orange-400", + "divide-orange-500", + "divide-orange-600", + "divide-orange-700", + "divide-orange-800", + "divide-orange-900", + "divide-orange-950", + "divide-amber-50", + "divide-amber-100", + "divide-amber-200", + "divide-amber-300", + "divide-amber-400", + "divide-amber-500", + "divide-amber-600", + "divide-amber-700", + "divide-amber-800", + "divide-amber-900", + "divide-amber-950", + "divide-yellow-50", + "divide-yellow-100", + "divide-yellow-200", + "divide-yellow-300", + "divide-yellow-400", + "divide-yellow-500", + "divide-yellow-600", + "divide-yellow-700", + "divide-yellow-800", + "divide-yellow-900", + "divide-yellow-950", + "divide-lime-50", + "divide-lime-100", + "divide-lime-200", + "divide-lime-300", + "divide-lime-400", + "divide-lime-500", + "divide-lime-600", + "divide-lime-700", + "divide-lime-800", + "divide-lime-900", + "divide-lime-950", + "divide-green-50", + "divide-green-100", + "divide-green-200", + "divide-green-300", + "divide-green-400", + "divide-green-500", + "divide-green-600", + "divide-green-700", + "divide-green-800", + "divide-green-900", + "divide-green-950", + "divide-emerald-50", + "divide-emerald-100", + "divide-emerald-200", + "divide-emerald-300", + "divide-emerald-400", + "divide-emerald-500", + "divide-emerald-600", + "divide-emerald-700", + "divide-emerald-800", + "divide-emerald-900", + "divide-emerald-950", + "divide-teal-50", + "divide-teal-100", + "divide-teal-200", + "divide-teal-300", + "divide-teal-400", + "divide-teal-500", + "divide-teal-600", + "divide-teal-700", + "divide-teal-800", + "divide-teal-900", + "divide-teal-950", + "divide-cyan-50", + "divide-cyan-100", + "divide-cyan-200", + "divide-cyan-300", + "divide-cyan-400", + "divide-cyan-500", + "divide-cyan-600", + "divide-cyan-700", + "divide-cyan-800", + "divide-cyan-900", + "divide-cyan-950", + "divide-sky-50", + "divide-sky-100", + "divide-sky-200", + "divide-sky-300", + "divide-sky-400", + "divide-sky-500", + "divide-sky-600", + "divide-sky-700", + "divide-sky-800", + "divide-sky-900", + "divide-sky-950", + "divide-blue-50", + "divide-blue-100", + "divide-blue-200", + "divide-blue-300", + "divide-blue-400", + "divide-blue-500", + "divide-blue-600", + "divide-blue-700", + "divide-blue-800", + "divide-blue-900", + "divide-blue-950", + "divide-indigo-50", + "divide-indigo-100", + "divide-indigo-200", + "divide-indigo-300", + "divide-indigo-400", + "divide-indigo-500", + "divide-indigo-600", + "divide-indigo-700", + "divide-indigo-800", + "divide-indigo-900", + "divide-indigo-950", + "divide-violet-50", + "divide-violet-100", + "divide-violet-200", + "divide-violet-300", + "divide-violet-400", + "divide-violet-500", + "divide-violet-600", + "divide-violet-700", + "divide-violet-800", + "divide-violet-900", + "divide-violet-950", + "divide-purple-50", + "divide-purple-100", + "divide-purple-200", + "divide-purple-300", + "divide-purple-400", + "divide-purple-500", + "divide-purple-600", + "divide-purple-700", + "divide-purple-800", + "divide-purple-900", + "divide-purple-950", + "divide-fuchsia-50", + "divide-fuchsia-100", + "divide-fuchsia-200", + "divide-fuchsia-300", + "divide-fuchsia-400", + "divide-fuchsia-500", + "divide-fuchsia-600", + "divide-fuchsia-700", + "divide-fuchsia-800", + "divide-fuchsia-900", + "divide-fuchsia-950", + "divide-pink-50", + "divide-pink-100", + "divide-pink-200", + "divide-pink-300", + "divide-pink-400", + "divide-pink-500", + "divide-pink-600", + "divide-pink-700", + "divide-pink-800", + "divide-pink-900", + "divide-pink-950", + "divide-rose-50", + "divide-rose-100", + "divide-rose-200", + "divide-rose-300", + "divide-rose-400", + "divide-rose-500", + "divide-rose-600", + "divide-rose-700", + "divide-rose-800", + "divide-rose-900", + "divide-rose-950", + "divide-solid", + "divide-dashed", + "divide-dotted", + "divide-double", + "divide-none", + "outline-0", + "outline-1", + "outline-2", + "outline-4", + "outline-8", + "outline-inherit", + "outline-current", + "outline-transparent", + "outline-black", + "outline-white", + "outline-slate-50", + "outline-slate-100", + "outline-slate-200", + "outline-slate-300", + "outline-slate-400", + "outline-slate-500", + "outline-slate-600", + "outline-slate-700", + "outline-slate-800", + "outline-slate-900", + "outline-slate-950", + "outline-gray-50", + "outline-gray-100", + "outline-gray-200", + "outline-gray-300", + "outline-gray-400", + "outline-gray-500", + "outline-gray-600", + "outline-gray-700", + "outline-gray-800", + "outline-gray-900", + "outline-gray-950", + "outline-zinc-50", + "outline-zinc-100", + "outline-zinc-200", + "outline-zinc-300", + "outline-zinc-400", + "outline-zinc-500", + "outline-zinc-600", + "outline-zinc-700", + "outline-zinc-800", + "outline-zinc-900", + "outline-zinc-950", + "outline-neutral-50", + "outline-neutral-100", + "outline-neutral-200", + "outline-neutral-300", + "outline-neutral-400", + "outline-neutral-500", + "outline-neutral-600", + "outline-neutral-700", + "outline-neutral-800", + "outline-neutral-900", + "outline-neutral-950", + "outline-stone-50", + "outline-stone-100", + "outline-stone-200", + "outline-stone-300", + "outline-stone-400", + "outline-stone-500", + "outline-stone-600", + "outline-stone-700", + "outline-stone-800", + "outline-stone-900", + "outline-stone-950", + "outline-red-50", + "outline-red-100", + "outline-red-200", + "outline-red-300", + "outline-red-400", + "outline-red-500", + "outline-red-600", + "outline-red-700", + "outline-red-800", + "outline-red-900", + "outline-red-950", + "outline-orange-50", + "outline-orange-100", + "outline-orange-200", + "outline-orange-300", + "outline-orange-400", + "outline-orange-500", + "outline-orange-600", + "outline-orange-700", + "outline-orange-800", + "outline-orange-900", + "outline-orange-950", + "outline-amber-50", + "outline-amber-100", + "outline-amber-200", + "outline-amber-300", + "outline-amber-400", + "outline-amber-500", + "outline-amber-600", + "outline-amber-700", + "outline-amber-800", + "outline-amber-900", + "outline-amber-950", + "outline-yellow-50", + "outline-yellow-100", + "outline-yellow-200", + "outline-yellow-300", + "outline-yellow-400", + "outline-yellow-500", + "outline-yellow-600", + "outline-yellow-700", + "outline-yellow-800", + "outline-yellow-900", + "outline-yellow-950", + "outline-lime-50", + "outline-lime-100", + "outline-lime-200", + "outline-lime-300", + "outline-lime-400", + "outline-lime-500", + "outline-lime-600", + "outline-lime-700", + "outline-lime-800", + "outline-lime-900", + "outline-lime-950", + "outline-green-50", + "outline-green-100", + "outline-green-200", + "outline-green-300", + "outline-green-400", + "outline-green-500", + "outline-green-600", + "outline-green-700", + "outline-green-800", + "outline-green-900", + "outline-green-950", + "outline-emerald-50", + "outline-emerald-100", + "outline-emerald-200", + "outline-emerald-300", + "outline-emerald-400", + "outline-emerald-500", + "outline-emerald-600", + "outline-emerald-700", + "outline-emerald-800", + "outline-emerald-900", + "outline-emerald-950", + "outline-teal-50", + "outline-teal-100", + "outline-teal-200", + "outline-teal-300", + "outline-teal-400", + "outline-teal-500", + "outline-teal-600", + "outline-teal-700", + "outline-teal-800", + "outline-teal-900", + "outline-teal-950", + "outline-cyan-50", + "outline-cyan-100", + "outline-cyan-200", + "outline-cyan-300", + "outline-cyan-400", + "outline-cyan-500", + "outline-cyan-600", + "outline-cyan-700", + "outline-cyan-800", + "outline-cyan-900", + "outline-cyan-950", + "outline-sky-50", + "outline-sky-100", + "outline-sky-200", + "outline-sky-300", + "outline-sky-400", + "outline-sky-500", + "outline-sky-600", + "outline-sky-700", + "outline-sky-800", + "outline-sky-900", + "outline-sky-950", + "outline-blue-50", + "outline-blue-100", + "outline-blue-200", + "outline-blue-300", + "outline-blue-400", + "outline-blue-500", + "outline-blue-600", + "outline-blue-700", + "outline-blue-800", + "outline-blue-900", + "outline-blue-950", + "outline-indigo-50", + "outline-indigo-100", + "outline-indigo-200", + "outline-indigo-300", + "outline-indigo-400", + "outline-indigo-500", + "outline-indigo-600", + "outline-indigo-700", + "outline-indigo-800", + "outline-indigo-900", + "outline-indigo-950", + "outline-violet-50", + "outline-violet-100", + "outline-violet-200", + "outline-violet-300", + "outline-violet-400", + "outline-violet-500", + "outline-violet-600", + "outline-violet-700", + "outline-violet-800", + "outline-violet-900", + "outline-violet-950", + "outline-purple-50", + "outline-purple-100", + "outline-purple-200", + "outline-purple-300", + "outline-purple-400", + "outline-purple-500", + "outline-purple-600", + "outline-purple-700", + "outline-purple-800", + "outline-purple-900", + "outline-purple-950", + "outline-fuchsia-50", + "outline-fuchsia-100", + "outline-fuchsia-200", + "outline-fuchsia-300", + "outline-fuchsia-400", + "outline-fuchsia-500", + "outline-fuchsia-600", + "outline-fuchsia-700", + "outline-fuchsia-800", + "outline-fuchsia-900", + "outline-fuchsia-950", + "outline-pink-50", + "outline-pink-100", + "outline-pink-200", + "outline-pink-300", + "outline-pink-400", + "outline-pink-500", + "outline-pink-600", + "outline-pink-700", + "outline-pink-800", + "outline-pink-900", + "outline-pink-950", + "outline-rose-50", + "outline-rose-100", + "outline-rose-200", + "outline-rose-300", + "outline-rose-400", + "outline-rose-500", + "outline-rose-600", + "outline-rose-700", + "outline-rose-800", + "outline-rose-900", + "outline-rose-950", + "outline-none", + "outline", + "outline-dashed", + "outline-dotted", + "outline-double", + "outline-offset-0", + "outline-offset-1", + "outline-offset-2", + "outline-offset-4", + "outline-offset-8", + "ring-0", + "ring-1", + "ring-2", + "ring", + "ring-4", + "ring-8", + "ring-inset", + "ring-inherit", + "ring-current", + "ring-transparent", + "ring-black", + "ring-white", + "ring-slate-50", + "ring-slate-100", + "ring-slate-200", + "ring-slate-300", + "ring-slate-400", + "ring-slate-500", + "ring-slate-600", + "ring-slate-700", + "ring-slate-800", + "ring-slate-900", + "ring-slate-950", + "ring-gray-50", + "ring-gray-100", + "ring-gray-200", + "ring-gray-300", + "ring-gray-400", + "ring-gray-500", + "ring-gray-600", + "ring-gray-700", + "ring-gray-800", + "ring-gray-900", + "ring-gray-950", + "ring-zinc-50", + "ring-zinc-100", + "ring-zinc-200", + "ring-zinc-300", + "ring-zinc-400", + "ring-zinc-500", + "ring-zinc-600", + "ring-zinc-700", + "ring-zinc-800", + "ring-zinc-900", + "ring-zinc-950", + "ring-neutral-50", + "ring-neutral-100", + "ring-neutral-200", + "ring-neutral-300", + "ring-neutral-400", + "ring-neutral-500", + "ring-neutral-600", + "ring-neutral-700", + "ring-neutral-800", + "ring-neutral-900", + "ring-neutral-950", + "ring-stone-50", + "ring-stone-100", + "ring-stone-200", + "ring-stone-300", + "ring-stone-400", + "ring-stone-500", + "ring-stone-600", + "ring-stone-700", + "ring-stone-800", + "ring-stone-900", + "ring-stone-950", + "ring-red-50", + "ring-red-100", + "ring-red-200", + "ring-red-300", + "ring-red-400", + "ring-red-500", + "ring-red-600", + "ring-red-700", + "ring-red-800", + "ring-red-900", + "ring-red-950", + "ring-orange-50", + "ring-orange-100", + "ring-orange-200", + "ring-orange-300", + "ring-orange-400", + "ring-orange-500", + "ring-orange-600", + "ring-orange-700", + "ring-orange-800", + "ring-orange-900", + "ring-orange-950", + "ring-amber-50", + "ring-amber-100", + "ring-amber-200", + "ring-amber-300", + "ring-amber-400", + "ring-amber-500", + "ring-amber-600", + "ring-amber-700", + "ring-amber-800", + "ring-amber-900", + "ring-amber-950", + "ring-yellow-50", + "ring-yellow-100", + "ring-yellow-200", + "ring-yellow-300", + "ring-yellow-400", + "ring-yellow-500", + "ring-yellow-600", + "ring-yellow-700", + "ring-yellow-800", + "ring-yellow-900", + "ring-yellow-950", + "ring-lime-50", + "ring-lime-100", + "ring-lime-200", + "ring-lime-300", + "ring-lime-400", + "ring-lime-500", + "ring-lime-600", + "ring-lime-700", + "ring-lime-800", + "ring-lime-900", + "ring-lime-950", + "ring-green-50", + "ring-green-100", + "ring-green-200", + "ring-green-300", + "ring-green-400", + "ring-green-500", + "ring-green-600", + "ring-green-700", + "ring-green-800", + "ring-green-900", + "ring-green-950", + "ring-emerald-50", + "ring-emerald-100", + "ring-emerald-200", + "ring-emerald-300", + "ring-emerald-400", + "ring-emerald-500", + "ring-emerald-600", + "ring-emerald-700", + "ring-emerald-800", + "ring-emerald-900", + "ring-emerald-950", + "ring-teal-50", + "ring-teal-100", + "ring-teal-200", + "ring-teal-300", + "ring-teal-400", + "ring-teal-500", + "ring-teal-600", + "ring-teal-700", + "ring-teal-800", + "ring-teal-900", + "ring-teal-950", + "ring-cyan-50", + "ring-cyan-100", + "ring-cyan-200", + "ring-cyan-300", + "ring-cyan-400", + "ring-cyan-500", + "ring-cyan-600", + "ring-cyan-700", + "ring-cyan-800", + "ring-cyan-900", + "ring-cyan-950", + "ring-sky-50", + "ring-sky-100", + "ring-sky-200", + "ring-sky-300", + "ring-sky-400", + "ring-sky-500", + "ring-sky-600", + "ring-sky-700", + "ring-sky-800", + "ring-sky-900", + "ring-sky-950", + "ring-blue-50", + "ring-blue-100", + "ring-blue-200", + "ring-blue-300", + "ring-blue-400", + "ring-blue-500", + "ring-blue-600", + "ring-blue-700", + "ring-blue-800", + "ring-blue-900", + "ring-blue-950", + "ring-indigo-50", + "ring-indigo-100", + "ring-indigo-200", + "ring-indigo-300", + "ring-indigo-400", + "ring-indigo-500", + "ring-indigo-600", + "ring-indigo-700", + "ring-indigo-800", + "ring-indigo-900", + "ring-indigo-950", + "ring-violet-50", + "ring-violet-100", + "ring-violet-200", + "ring-violet-300", + "ring-violet-400", + "ring-violet-500", + "ring-violet-600", + "ring-violet-700", + "ring-violet-800", + "ring-violet-900", + "ring-violet-950", + "ring-purple-50", + "ring-purple-100", + "ring-purple-200", + "ring-purple-300", + "ring-purple-400", + "ring-purple-500", + "ring-purple-600", + "ring-purple-700", + "ring-purple-800", + "ring-purple-900", + "ring-purple-950", + "ring-fuchsia-50", + "ring-fuchsia-100", + "ring-fuchsia-200", + "ring-fuchsia-300", + "ring-fuchsia-400", + "ring-fuchsia-500", + "ring-fuchsia-600", + "ring-fuchsia-700", + "ring-fuchsia-800", + "ring-fuchsia-900", + "ring-fuchsia-950", + "ring-pink-50", + "ring-pink-100", + "ring-pink-200", + "ring-pink-300", + "ring-pink-400", + "ring-pink-500", + "ring-pink-600", + "ring-pink-700", + "ring-pink-800", + "ring-pink-900", + "ring-pink-950", + "ring-rose-50", + "ring-rose-100", + "ring-rose-200", + "ring-rose-300", + "ring-rose-400", + "ring-rose-500", + "ring-rose-600", + "ring-rose-700", + "ring-rose-800", + "ring-rose-900", + "ring-rose-950", + "ring-offset-0", + "ring-offset-1", + "ring-offset-2", + "ring-offset-4", + "ring-offset-8", + "ring-offset-inherit", + "ring-offset-current", + "ring-offset-transparent", + "ring-offset-black", + "ring-offset-white", + "ring-offset-slate-50", + "ring-offset-slate-100", + "ring-offset-slate-200", + "ring-offset-slate-300", + "ring-offset-slate-400", + "ring-offset-slate-500", + "ring-offset-slate-600", + "ring-offset-slate-700", + "ring-offset-slate-800", + "ring-offset-slate-900", + "ring-offset-slate-950", + "ring-offset-gray-50", + "ring-offset-gray-100", + "ring-offset-gray-200", + "ring-offset-gray-300", + "ring-offset-gray-400", + "ring-offset-gray-500", + "ring-offset-gray-600", + "ring-offset-gray-700", + "ring-offset-gray-800", + "ring-offset-gray-900", + "ring-offset-gray-950", + "ring-offset-zinc-50", + "ring-offset-zinc-100", + "ring-offset-zinc-200", + "ring-offset-zinc-300", + "ring-offset-zinc-400", + "ring-offset-zinc-500", + "ring-offset-zinc-600", + "ring-offset-zinc-700", + "ring-offset-zinc-800", + "ring-offset-zinc-900", + "ring-offset-zinc-950", + "ring-offset-neutral-50", + "ring-offset-neutral-100", + "ring-offset-neutral-200", + "ring-offset-neutral-300", + "ring-offset-neutral-400", + "ring-offset-neutral-500", + "ring-offset-neutral-600", + "ring-offset-neutral-700", + "ring-offset-neutral-800", + "ring-offset-neutral-900", + "ring-offset-neutral-950", + "ring-offset-stone-50", + "ring-offset-stone-100", + "ring-offset-stone-200", + "ring-offset-stone-300", + "ring-offset-stone-400", + "ring-offset-stone-500", + "ring-offset-stone-600", + "ring-offset-stone-700", + "ring-offset-stone-800", + "ring-offset-stone-900", + "ring-offset-stone-950", + "ring-offset-red-50", + "ring-offset-red-100", + "ring-offset-red-200", + "ring-offset-red-300", + "ring-offset-red-400", + "ring-offset-red-500", + "ring-offset-red-600", + "ring-offset-red-700", + "ring-offset-red-800", + "ring-offset-red-900", + "ring-offset-red-950", + "ring-offset-orange-50", + "ring-offset-orange-100", + "ring-offset-orange-200", + "ring-offset-orange-300", + "ring-offset-orange-400", + "ring-offset-orange-500", + "ring-offset-orange-600", + "ring-offset-orange-700", + "ring-offset-orange-800", + "ring-offset-orange-900", + "ring-offset-orange-950", + "ring-offset-amber-50", + "ring-offset-amber-100", + "ring-offset-amber-200", + "ring-offset-amber-300", + "ring-offset-amber-400", + "ring-offset-amber-500", + "ring-offset-amber-600", + "ring-offset-amber-700", + "ring-offset-amber-800", + "ring-offset-amber-900", + "ring-offset-amber-950", + "ring-offset-yellow-50", + "ring-offset-yellow-100", + "ring-offset-yellow-200", + "ring-offset-yellow-300", + "ring-offset-yellow-400", + "ring-offset-yellow-500", + "ring-offset-yellow-600", + "ring-offset-yellow-700", + "ring-offset-yellow-800", + "ring-offset-yellow-900", + "ring-offset-yellow-950", + "ring-offset-lime-50", + "ring-offset-lime-100", + "ring-offset-lime-200", + "ring-offset-lime-300", + "ring-offset-lime-400", + "ring-offset-lime-500", + "ring-offset-lime-600", + "ring-offset-lime-700", + "ring-offset-lime-800", + "ring-offset-lime-900", + "ring-offset-lime-950", + "ring-offset-green-50", + "ring-offset-green-100", + "ring-offset-green-200", + "ring-offset-green-300", + "ring-offset-green-400", + "ring-offset-green-500", + "ring-offset-green-600", + "ring-offset-green-700", + "ring-offset-green-800", + "ring-offset-green-900", + "ring-offset-green-950", + "ring-offset-emerald-50", + "ring-offset-emerald-100", + "ring-offset-emerald-200", + "ring-offset-emerald-300", + "ring-offset-emerald-400", + "ring-offset-emerald-500", + "ring-offset-emerald-600", + "ring-offset-emerald-700", + "ring-offset-emerald-800", + "ring-offset-emerald-900", + "ring-offset-emerald-950", + "ring-offset-teal-50", + "ring-offset-teal-100", + "ring-offset-teal-200", + "ring-offset-teal-300", + "ring-offset-teal-400", + "ring-offset-teal-500", + "ring-offset-teal-600", + "ring-offset-teal-700", + "ring-offset-teal-800", + "ring-offset-teal-900", + "ring-offset-teal-950", + "ring-offset-cyan-50", + "ring-offset-cyan-100", + "ring-offset-cyan-200", + "ring-offset-cyan-300", + "ring-offset-cyan-400", + "ring-offset-cyan-500", + "ring-offset-cyan-600", + "ring-offset-cyan-700", + "ring-offset-cyan-800", + "ring-offset-cyan-900", + "ring-offset-cyan-950", + "ring-offset-sky-50", + "ring-offset-sky-100", + "ring-offset-sky-200", + "ring-offset-sky-300", + "ring-offset-sky-400", + "ring-offset-sky-500", + "ring-offset-sky-600", + "ring-offset-sky-700", + "ring-offset-sky-800", + "ring-offset-sky-900", + "ring-offset-sky-950", + "ring-offset-blue-50", + "ring-offset-blue-100", + "ring-offset-blue-200", + "ring-offset-blue-300", + "ring-offset-blue-400", + "ring-offset-blue-500", + "ring-offset-blue-600", + "ring-offset-blue-700", + "ring-offset-blue-800", + "ring-offset-blue-900", + "ring-offset-blue-950", + "ring-offset-indigo-50", + "ring-offset-indigo-100", + "ring-offset-indigo-200", + "ring-offset-indigo-300", + "ring-offset-indigo-400", + "ring-offset-indigo-500", + "ring-offset-indigo-600", + "ring-offset-indigo-700", + "ring-offset-indigo-800", + "ring-offset-indigo-900", + "ring-offset-indigo-950", + "ring-offset-violet-50", + "ring-offset-violet-100", + "ring-offset-violet-200", + "ring-offset-violet-300", + "ring-offset-violet-400", + "ring-offset-violet-500", + "ring-offset-violet-600", + "ring-offset-violet-700", + "ring-offset-violet-800", + "ring-offset-violet-900", + "ring-offset-violet-950", + "ring-offset-purple-50", + "ring-offset-purple-100", + "ring-offset-purple-200", + "ring-offset-purple-300", + "ring-offset-purple-400", + "ring-offset-purple-500", + "ring-offset-purple-600", + "ring-offset-purple-700", + "ring-offset-purple-800", + "ring-offset-purple-900", + "ring-offset-purple-950", + "ring-offset-fuchsia-50", + "ring-offset-fuchsia-100", + "ring-offset-fuchsia-200", + "ring-offset-fuchsia-300", + "ring-offset-fuchsia-400", + "ring-offset-fuchsia-500", + "ring-offset-fuchsia-600", + "ring-offset-fuchsia-700", + "ring-offset-fuchsia-800", + "ring-offset-fuchsia-900", + "ring-offset-fuchsia-950", + "ring-offset-pink-50", + "ring-offset-pink-100", + "ring-offset-pink-200", + "ring-offset-pink-300", + "ring-offset-pink-400", + "ring-offset-pink-500", + "ring-offset-pink-600", + "ring-offset-pink-700", + "ring-offset-pink-800", + "ring-offset-pink-900", + "ring-offset-pink-950", + "ring-offset-rose-50", + "ring-offset-rose-100", + "ring-offset-rose-200", + "ring-offset-rose-300", + "ring-offset-rose-400", + "ring-offset-rose-500", + "ring-offset-rose-600", + "ring-offset-rose-700", + "ring-offset-rose-800", + "ring-offset-rose-900", + "ring-offset-rose-950", + "shadow-sm", + "shadow", + "shadow-md", + "shadow-lg", + "shadow-xl", + "shadow-2xl", + "shadow-inner", + "shadow-none", + "shadow-inherit", + "shadow-current", + "shadow-transparent", + "shadow-black", + "shadow-white", + "shadow-slate-50", + "shadow-slate-100", + "shadow-slate-200", + "shadow-slate-300", + "shadow-slate-400", + "shadow-slate-500", + "shadow-slate-600", + "shadow-slate-700", + "shadow-slate-800", + "shadow-slate-900", + "shadow-slate-950", + "shadow-gray-50", + "shadow-gray-100", + "shadow-gray-200", + "shadow-gray-300", + "shadow-gray-400", + "shadow-gray-500", + "shadow-gray-600", + "shadow-gray-700", + "shadow-gray-800", + "shadow-gray-900", + "shadow-gray-950", + "shadow-zinc-50", + "shadow-zinc-100", + "shadow-zinc-200", + "shadow-zinc-300", + "shadow-zinc-400", + "shadow-zinc-500", + "shadow-zinc-600", + "shadow-zinc-700", + "shadow-zinc-800", + "shadow-zinc-900", + "shadow-zinc-950", + "shadow-neutral-50", + "shadow-neutral-100", + "shadow-neutral-200", + "shadow-neutral-300", + "shadow-neutral-400", + "shadow-neutral-500", + "shadow-neutral-600", + "shadow-neutral-700", + "shadow-neutral-800", + "shadow-neutral-900", + "shadow-neutral-950", + "shadow-stone-50", + "shadow-stone-100", + "shadow-stone-200", + "shadow-stone-300", + "shadow-stone-400", + "shadow-stone-500", + "shadow-stone-600", + "shadow-stone-700", + "shadow-stone-800", + "shadow-stone-900", + "shadow-stone-950", + "shadow-red-50", + "shadow-red-100", + "shadow-red-200", + "shadow-red-300", + "shadow-red-400", + "shadow-red-500", + "shadow-red-600", + "shadow-red-700", + "shadow-red-800", + "shadow-red-900", + "shadow-red-950", + "shadow-orange-50", + "shadow-orange-100", + "shadow-orange-200", + "shadow-orange-300", + "shadow-orange-400", + "shadow-orange-500", + "shadow-orange-600", + "shadow-orange-700", + "shadow-orange-800", + "shadow-orange-900", + "shadow-orange-950", + "shadow-amber-50", + "shadow-amber-100", + "shadow-amber-200", + "shadow-amber-300", + "shadow-amber-400", + "shadow-amber-500", + "shadow-amber-600", + "shadow-amber-700", + "shadow-amber-800", + "shadow-amber-900", + "shadow-amber-950", + "shadow-yellow-50", + "shadow-yellow-100", + "shadow-yellow-200", + "shadow-yellow-300", + "shadow-yellow-400", + "shadow-yellow-500", + "shadow-yellow-600", + "shadow-yellow-700", + "shadow-yellow-800", + "shadow-yellow-900", + "shadow-yellow-950", + "shadow-lime-50", + "shadow-lime-100", + "shadow-lime-200", + "shadow-lime-300", + "shadow-lime-400", + "shadow-lime-500", + "shadow-lime-600", + "shadow-lime-700", + "shadow-lime-800", + "shadow-lime-900", + "shadow-lime-950", + "shadow-green-50", + "shadow-green-100", + "shadow-green-200", + "shadow-green-300", + "shadow-green-400", + "shadow-green-500", + "shadow-green-600", + "shadow-green-700", + "shadow-green-800", + "shadow-green-900", + "shadow-green-950", + "shadow-emerald-50", + "shadow-emerald-100", + "shadow-emerald-200", + "shadow-emerald-300", + "shadow-emerald-400", + "shadow-emerald-500", + "shadow-emerald-600", + "shadow-emerald-700", + "shadow-emerald-800", + "shadow-emerald-900", + "shadow-emerald-950", + "shadow-teal-50", + "shadow-teal-100", + "shadow-teal-200", + "shadow-teal-300", + "shadow-teal-400", + "shadow-teal-500", + "shadow-teal-600", + "shadow-teal-700", + "shadow-teal-800", + "shadow-teal-900", + "shadow-teal-950", + "shadow-cyan-50", + "shadow-cyan-100", + "shadow-cyan-200", + "shadow-cyan-300", + "shadow-cyan-400", + "shadow-cyan-500", + "shadow-cyan-600", + "shadow-cyan-700", + "shadow-cyan-800", + "shadow-cyan-900", + "shadow-cyan-950", + "shadow-sky-50", + "shadow-sky-100", + "shadow-sky-200", + "shadow-sky-300", + "shadow-sky-400", + "shadow-sky-500", + "shadow-sky-600", + "shadow-sky-700", + "shadow-sky-800", + "shadow-sky-900", + "shadow-sky-950", + "shadow-blue-50", + "shadow-blue-100", + "shadow-blue-200", + "shadow-blue-300", + "shadow-blue-400", + "shadow-blue-500", + "shadow-blue-600", + "shadow-blue-700", + "shadow-blue-800", + "shadow-blue-900", + "shadow-blue-950", + "shadow-indigo-50", + "shadow-indigo-100", + "shadow-indigo-200", + "shadow-indigo-300", + "shadow-indigo-400", + "shadow-indigo-500", + "shadow-indigo-600", + "shadow-indigo-700", + "shadow-indigo-800", + "shadow-indigo-900", + "shadow-indigo-950", + "shadow-violet-50", + "shadow-violet-100", + "shadow-violet-200", + "shadow-violet-300", + "shadow-violet-400", + "shadow-violet-500", + "shadow-violet-600", + "shadow-violet-700", + "shadow-violet-800", + "shadow-violet-900", + "shadow-violet-950", + "shadow-purple-50", + "shadow-purple-100", + "shadow-purple-200", + "shadow-purple-300", + "shadow-purple-400", + "shadow-purple-500", + "shadow-purple-600", + "shadow-purple-700", + "shadow-purple-800", + "shadow-purple-900", + "shadow-purple-950", + "shadow-fuchsia-50", + "shadow-fuchsia-100", + "shadow-fuchsia-200", + "shadow-fuchsia-300", + "shadow-fuchsia-400", + "shadow-fuchsia-500", + "shadow-fuchsia-600", + "shadow-fuchsia-700", + "shadow-fuchsia-800", + "shadow-fuchsia-900", + "shadow-fuchsia-950", + "shadow-pink-50", + "shadow-pink-100", + "shadow-pink-200", + "shadow-pink-300", + "shadow-pink-400", + "shadow-pink-500", + "shadow-pink-600", + "shadow-pink-700", + "shadow-pink-800", + "shadow-pink-900", + "shadow-pink-950", + "shadow-rose-50", + "shadow-rose-100", + "shadow-rose-200", + "shadow-rose-300", + "shadow-rose-400", + "shadow-rose-500", + "shadow-rose-600", + "shadow-rose-700", + "shadow-rose-800", + "shadow-rose-900", + "shadow-rose-950", + "opacity-0", + "opacity-5", + "opacity-10", + "opacity-15", + "opacity-20", + "opacity-25", + "opacity-30", + "opacity-35", + "opacity-40", + "opacity-45", + "opacity-50", + "opacity-55", + "opacity-60", + "opacity-65", + "opacity-70", + "opacity-75", + "opacity-80", + "opacity-85", + "opacity-90", + "opacity-95", + "opacity-100", + "mix-blend-normal", + "mix-blend-multiply", + "mix-blend-screen", + "mix-blend-overlay", + "mix-blend-darken", + "mix-blend-lighten", + "mix-blend-color-dodge", + "mix-blend-color-burn", + "mix-blend-hard-light", + "mix-blend-soft-light", + "mix-blend-difference", + "mix-blend-exclusion", + "mix-blend-hue", + "mix-blend-saturation", + "mix-blend-color", + "mix-blend-luminosity", + "mix-blend-plus-darker", + "mix-blend-plus-lighter", + "bg-blend-normal", + "bg-blend-multiply", + "bg-blend-screen", + "bg-blend-overlay", + "bg-blend-darken", + "bg-blend-lighten", + "bg-blend-color-dodge", + "bg-blend-color-burn", + "bg-blend-hard-light", + "bg-blend-soft-light", + "bg-blend-difference", + "bg-blend-exclusion", + "bg-blend-hue", + "bg-blend-saturation", + "bg-blend-color", + "bg-blend-luminosity", + "blur-none", + "blur-sm", + "blur", + "blur-md", + "blur-lg", + "blur-xl", + "blur-2xl", + "blur-3xl", + "brightness-0", + "brightness-50", + "brightness-75", + "brightness-90", + "brightness-95", + "brightness-100", + "brightness-105", + "brightness-110", + "brightness-125", + "brightness-150", + "brightness-200", + "contrast-0", + "contrast-50", + "contrast-75", + "contrast-100", + "contrast-125", + "contrast-150", + "contrast-200", + "drop-shadow-sm", + "drop-shadow", + "drop-shadow-md", + "drop-shadow-lg", + "drop-shadow-xl", + "drop-shadow-2xl", + "drop-shadow-none", + "grayscale-0", + "grayscale", + "hue-rotate-0", + "hue-rotate-15", + "hue-rotate-30", + "hue-rotate-60", + "hue-rotate-90", + "hue-rotate-180", + "invert-0", + "invert", + "saturate-0", + "saturate-50", + "saturate-100", + "saturate-150", + "saturate-200", + "sepia-0", + "sepia", + "backdrop-blur-none", + "backdrop-blur-sm", + "backdrop-blur", + "backdrop-blur-md", + "backdrop-blur-lg", + "backdrop-blur-xl", + "backdrop-blur-2xl", + "backdrop-blur-3xl", + "backdrop-brightness-0", + "backdrop-brightness-50", + "backdrop-brightness-75", + "backdrop-brightness-90", + "backdrop-brightness-95", + "backdrop-brightness-100", + "backdrop-brightness-105", + "backdrop-brightness-110", + "backdrop-brightness-125", + "backdrop-brightness-150", + "backdrop-brightness-200", + "backdrop-contrast-0", + "backdrop-contrast-50", + "backdrop-contrast-75", + "backdrop-contrast-100", + "backdrop-contrast-125", + "backdrop-contrast-150", + "backdrop-contrast-200", + "backdrop-grayscale-0", + "backdrop-grayscale", + "backdrop-hue-rotate-0", + "backdrop-hue-rotate-15", + "backdrop-hue-rotate-30", + "backdrop-hue-rotate-60", + "backdrop-hue-rotate-90", + "backdrop-hue-rotate-180", + "backdrop-invert-0", + "backdrop-invert", + "backdrop-opacity-0", + "backdrop-opacity-5", + "backdrop-opacity-10", + "backdrop-opacity-15", + "backdrop-opacity-20", + "backdrop-opacity-25", + "backdrop-opacity-30", + "backdrop-opacity-35", + "backdrop-opacity-40", + "backdrop-opacity-45", + "backdrop-opacity-50", + "backdrop-opacity-55", + "backdrop-opacity-60", + "backdrop-opacity-65", + "backdrop-opacity-70", + "backdrop-opacity-75", + "backdrop-opacity-80", + "backdrop-opacity-85", + "backdrop-opacity-90", + "backdrop-opacity-95", + "backdrop-opacity-100", + "backdrop-saturate-0", + "backdrop-saturate-50", + "backdrop-saturate-100", + "backdrop-saturate-150", + "backdrop-saturate-200", + "backdrop-sepia-0", + "backdrop-sepia", + "border-collapse", + "border-separate", + "Indiana", + "Ohio", + "Michigan", + "Indiana", + "Ohio", + "Michigan", + "border-spacing-0", + "border-spacing-x-0", + "border-spacing-y-0", + "border-spacing-px", + "border-spacing-x-px", + "border-spacing-y-px", + "border-spacing-0.5", + "border-spacing-x-0.5", + "border-spacing-y-0.5", + "border-spacing-1", + "border-spacing-x-1", + "border-spacing-y-1", + "border-spacing-1.5", + "border-spacing-x-1.5", + "border-spacing-y-1.5", + "border-spacing-2", + "border-spacing-x-2", + "border-spacing-y-2", + "border-spacing-2.5", + "border-spacing-x-2.5", + "border-spacing-y-2.5", + "border-spacing-3", + "border-spacing-x-3", + "border-spacing-y-3", + "border-spacing-3.5", + "border-spacing-x-3.5", + "border-spacing-y-3.5", + "border-spacing-4", + "border-spacing-x-4", + "border-spacing-y-4", + "border-spacing-5", + "border-spacing-x-5", + "border-spacing-y-5", + "border-spacing-6", + "border-spacing-x-6", + "border-spacing-y-6", + "border-spacing-7", + "border-spacing-x-7", + "border-spacing-y-7", + "border-spacing-8", + "border-spacing-x-8", + "border-spacing-y-8", + "border-spacing-9", + "border-spacing-x-9", + "border-spacing-y-9", + "border-spacing-10", + "border-spacing-x-10", + "border-spacing-y-10", + "border-spacing-11", + "border-spacing-x-11", + "border-spacing-y-11", + "border-spacing-12", + "border-spacing-x-12", + "border-spacing-y-12", + "border-spacing-14", + "border-spacing-x-14", + "border-spacing-y-14", + "border-spacing-16", + "border-spacing-x-16", + "border-spacing-y-16", + "border-spacing-20", + "border-spacing-x-20", + "border-spacing-y-20", + "border-spacing-24", + "border-spacing-x-24", + "border-spacing-y-24", + "border-spacing-28", + "border-spacing-x-28", + "border-spacing-y-28", + "border-spacing-32", + "border-spacing-x-32", + "border-spacing-y-32", + "border-spacing-36", + "border-spacing-x-36", + "border-spacing-y-36", + "border-spacing-40", + "border-spacing-x-40", + "border-spacing-y-40", + "border-spacing-44", + "border-spacing-x-44", + "border-spacing-y-44", + "border-spacing-48", + "border-spacing-x-48", + "border-spacing-y-48", + "border-spacing-52", + "border-spacing-x-52", + "border-spacing-y-52", + "border-spacing-56", + "border-spacing-x-56", + "border-spacing-y-56", + "border-spacing-60", + "border-spacing-x-60", + "border-spacing-y-60", + "border-spacing-64", + "border-spacing-x-64", + "border-spacing-y-64", + "border-spacing-72", + "border-spacing-x-72", + "border-spacing-y-72", + "border-spacing-80", + "border-spacing-x-80", + "border-spacing-y-80", + "border-spacing-96", + "border-spacing-x-96", + "border-spacing-y-96", + "Indiana", + "Ohio", + "Michigan", + "table-auto", + "table-fixed", + "caption-top", + "caption-bottom", + "transition-none", + "transition-all", + "transition", + "transition-colors", + "transition-opacity", + "transition-shadow", + "transition-transform", + "duration-0", + "duration-75", + "duration-100", + "duration-150", + "duration-200", + "duration-300", + "duration-500", + "duration-700", + "duration-1000", + "ease-linear", + "ease-in", + "ease-out", + "ease-in-out", + "delay-0", + "delay-75", + "delay-100", + "delay-150", + "delay-200", + "delay-300", + "delay-500", + "delay-700", + "delay-1000", + "animate-none", + "animate-spin", + "animate-ping", + "animate-pulse", + "animate-bounce", + "scale-0", + "scale-x-0", + "scale-y-0", + "scale-50", + "scale-x-50", + "scale-y-50", + "scale-75", + "scale-x-75", + "scale-y-75", + "scale-90", + "scale-x-90", + "scale-y-90", + "scale-95", + "scale-x-95", + "scale-y-95", + "scale-100", + "scale-x-100", + "scale-y-100", + "scale-105", + "scale-x-105", + "scale-y-105", + "scale-110", + "scale-x-110", + "scale-y-110", + "scale-125", + "scale-x-125", + "scale-y-125", + "scale-150", + "scale-x-150", + "scale-y-150", + "rotate-0", + "rotate-1", + "rotate-2", + "rotate-3", + "rotate-6", + "rotate-12", + "rotate-45", + "rotate-90", + "rotate-180", + "translate-x-0", + "translate-y-0", + "translate-x-px", + "translate-y-px", + "translate-x-0.5", + "translate-y-0.5", + "translate-x-1", + "translate-y-1", + "translate-x-1.5", + "translate-y-1.5", + "translate-x-2", + "translate-y-2", + "translate-x-2.5", + "translate-y-2.5", + "translate-x-3", + "translate-y-3", + "translate-x-3.5", + "translate-y-3.5", + "translate-x-4", + "translate-y-4", + "translate-x-5", + "translate-y-5", + "translate-x-6", + "translate-y-6", + "translate-x-7", + "translate-y-7", + "translate-x-8", + "translate-y-8", + "translate-x-9", + "translate-y-9", + "translate-x-10", + "translate-y-10", + "translate-x-11", + "translate-y-11", + "translate-x-12", + "translate-y-12", + "translate-x-14", + "translate-y-14", + "translate-x-16", + "translate-y-16", + "translate-x-20", + "translate-y-20", + "translate-x-24", + "translate-y-24", + "translate-x-28", + "translate-y-28", + "translate-x-32", + "translate-y-32", + "translate-x-36", + "translate-y-36", + "translate-x-40", + "translate-y-40", + "translate-x-44", + "translate-y-44", + "translate-x-48", + "translate-y-48", + "translate-x-52", + "translate-y-52", + "translate-x-56", + "translate-y-56", + "translate-x-60", + "translate-y-60", + "translate-x-64", + "translate-y-64", + "translate-x-72", + "translate-y-72", + "translate-x-80", + "translate-y-80", + "translate-x-96", + "translate-y-96", + "translate-x-1/2", + "translate-x-1/3", + "translate-x-2/3", + "translate-x-1/4", + "translate-x-2/4", + "translate-x-3/4", + "translate-x-full", + "translate-y-1/2", + "translate-y-1/3", + "translate-y-2/3", + "translate-y-1/4", + "translate-y-2/4", + "translate-y-3/4", + "translate-y-full", + "skew-x-0", + "skew-y-0", + "skew-x-1", + "skew-y-1", + "skew-x-2", + "skew-y-2", + "skew-x-3", + "skew-y-3", + "skew-x-6", + "skew-y-6", + "skew-x-12", + "skew-y-12", + "origin-center", + "origin-top", + "origin-top-right", + "origin-right", + "origin-bottom-right", + "origin-bottom", + "origin-bottom-left", + "origin-left", + "origin-top-left", + "accent-inherit", + "accent-current", + "accent-transparent", + "accent-black", + "accent-white", + "accent-slate-50", + "accent-slate-100", + "accent-slate-200", + "accent-slate-300", + "accent-slate-400", + "accent-slate-500", + "accent-slate-600", + "accent-slate-700", + "accent-slate-800", + "accent-slate-900", + "accent-slate-950", + "accent-gray-50", + "accent-gray-100", + "accent-gray-200", + "accent-gray-300", + "accent-gray-400", + "accent-gray-500", + "accent-gray-600", + "accent-gray-700", + "accent-gray-800", + "accent-gray-900", + "accent-gray-950", + "accent-zinc-50", + "accent-zinc-100", + "accent-zinc-200", + "accent-zinc-300", + "accent-zinc-400", + "accent-zinc-500", + "accent-zinc-600", + "accent-zinc-700", + "accent-zinc-800", + "accent-zinc-900", + "accent-zinc-950", + "accent-neutral-50", + "accent-neutral-100", + "accent-neutral-200", + "accent-neutral-300", + "accent-neutral-400", + "accent-neutral-500", + "accent-neutral-600", + "accent-neutral-700", + "accent-neutral-800", + "accent-neutral-900", + "accent-neutral-950", + "accent-stone-50", + "accent-stone-100", + "accent-stone-200", + "accent-stone-300", + "accent-stone-400", + "accent-stone-500", + "accent-stone-600", + "accent-stone-700", + "accent-stone-800", + "accent-stone-900", + "accent-stone-950", + "accent-red-50", + "accent-red-100", + "accent-red-200", + "accent-red-300", + "accent-red-400", + "accent-red-500", + "accent-red-600", + "accent-red-700", + "accent-red-800", + "accent-red-900", + "accent-red-950", + "accent-orange-50", + "accent-orange-100", + "accent-orange-200", + "accent-orange-300", + "accent-orange-400", + "accent-orange-500", + "accent-orange-600", + "accent-orange-700", + "accent-orange-800", + "accent-orange-900", + "accent-orange-950", + "accent-amber-50", + "accent-amber-100", + "accent-amber-200", + "accent-amber-300", + "accent-amber-400", + "accent-amber-500", + "accent-amber-600", + "accent-amber-700", + "accent-amber-800", + "accent-amber-900", + "accent-amber-950", + "accent-yellow-50", + "accent-yellow-100", + "accent-yellow-200", + "accent-yellow-300", + "accent-yellow-400", + "accent-yellow-500", + "accent-yellow-600", + "accent-yellow-700", + "accent-yellow-800", + "accent-yellow-900", + "accent-yellow-950", + "accent-lime-50", + "accent-lime-100", + "accent-lime-200", + "accent-lime-300", + "accent-lime-400", + "accent-lime-500", + "accent-lime-600", + "accent-lime-700", + "accent-lime-800", + "accent-lime-900", + "accent-lime-950", + "accent-green-50", + "accent-green-100", + "accent-green-200", + "accent-green-300", + "accent-green-400", + "accent-green-500", + "accent-green-600", + "accent-green-700", + "accent-green-800", + "accent-green-900", + "accent-green-950", + "accent-emerald-50", + "accent-emerald-100", + "accent-emerald-200", + "accent-emerald-300", + "accent-emerald-400", + "accent-emerald-500", + "accent-emerald-600", + "accent-emerald-700", + "accent-emerald-800", + "accent-emerald-900", + "accent-emerald-950", + "accent-teal-50", + "accent-teal-100", + "accent-teal-200", + "accent-teal-300", + "accent-teal-400", + "accent-teal-500", + "accent-teal-600", + "accent-teal-700", + "accent-teal-800", + "accent-teal-900", + "accent-teal-950", + "accent-cyan-50", + "accent-cyan-100", + "accent-cyan-200", + "accent-cyan-300", + "accent-cyan-400", + "accent-cyan-500", + "accent-cyan-600", + "accent-cyan-700", + "accent-cyan-800", + "accent-cyan-900", + "accent-cyan-950", + "accent-sky-50", + "accent-sky-100", + "accent-sky-200", + "accent-sky-300", + "accent-sky-400", + "accent-sky-500", + "accent-sky-600", + "accent-sky-700", + "accent-sky-800", + "accent-sky-900", + "accent-sky-950", + "accent-blue-50", + "accent-blue-100", + "accent-blue-200", + "accent-blue-300", + "accent-blue-400", + "accent-blue-500", + "accent-blue-600", + "accent-blue-700", + "accent-blue-800", + "accent-blue-900", + "accent-blue-950", + "accent-indigo-50", + "accent-indigo-100", + "accent-indigo-200", + "accent-indigo-300", + "accent-indigo-400", + "accent-indigo-500", + "accent-indigo-600", + "accent-indigo-700", + "accent-indigo-800", + "accent-indigo-900", + "accent-indigo-950", + "accent-violet-50", + "accent-violet-100", + "accent-violet-200", + "accent-violet-300", + "accent-violet-400", + "accent-violet-500", + "accent-violet-600", + "accent-violet-700", + "accent-violet-800", + "accent-violet-900", + "accent-violet-950", + "accent-purple-50", + "accent-purple-100", + "accent-purple-200", + "accent-purple-300", + "accent-purple-400", + "accent-purple-500", + "accent-purple-600", + "accent-purple-700", + "accent-purple-800", + "accent-purple-900", + "accent-purple-950", + "accent-fuchsia-50", + "accent-fuchsia-100", + "accent-fuchsia-200", + "accent-fuchsia-300", + "accent-fuchsia-400", + "accent-fuchsia-500", + "accent-fuchsia-600", + "accent-fuchsia-700", + "accent-fuchsia-800", + "accent-fuchsia-900", + "accent-fuchsia-950", + "accent-pink-50", + "accent-pink-100", + "accent-pink-200", + "accent-pink-300", + "accent-pink-400", + "accent-pink-500", + "accent-pink-600", + "accent-pink-700", + "accent-pink-800", + "accent-pink-900", + "accent-pink-950", + "accent-rose-50", + "accent-rose-100", + "accent-rose-200", + "accent-rose-300", + "accent-rose-400", + "accent-rose-500", + "accent-rose-600", + "accent-rose-700", + "accent-rose-800", + "accent-rose-900", + "accent-rose-950", + "accent-auto", + "appearance-none", + "appearance-auto", + "cursor-auto", + "cursor-default", + "cursor-pointer", + "cursor-wait", + "cursor-text", + "cursor-move", + "cursor-help", + "cursor-not-allowed", + "cursor-none", + "cursor-context-menu", + "cursor-progress", + "cursor-cell", + "cursor-crosshair", + "cursor-vertical-text", + "cursor-alias", + "cursor-copy", + "cursor-no-drop", + "cursor-grab", + "cursor-grabbing", + "cursor-all-scroll", + "cursor-col-resize", + "cursor-row-resize", + "cursor-n-resize", + "cursor-e-resize", + "cursor-s-resize", + "cursor-w-resize", + "cursor-ne-resize", + "cursor-nw-resize", + "cursor-se-resize", + "cursor-sw-resize", + "cursor-ew-resize", + "cursor-ns-resize", + "cursor-nesw-resize", + "cursor-nwse-resize", + "cursor-zoom-in", + "cursor-zoom-out", + "caret-inherit", + "caret-current", + "caret-transparent", + "caret-black", + "caret-white", + "caret-slate-50", + "caret-slate-100", + "caret-slate-200", + "caret-slate-300", + "caret-slate-400", + "caret-slate-500", + "caret-slate-600", + "caret-slate-700", + "caret-slate-800", + "caret-slate-900", + "caret-slate-950", + "caret-gray-50", + "caret-gray-100", + "caret-gray-200", + "caret-gray-300", + "caret-gray-400", + "caret-gray-500", + "caret-gray-600", + "caret-gray-700", + "caret-gray-800", + "caret-gray-900", + "caret-gray-950", + "caret-zinc-50", + "caret-zinc-100", + "caret-zinc-200", + "caret-zinc-300", + "caret-zinc-400", + "caret-zinc-500", + "caret-zinc-600", + "caret-zinc-700", + "caret-zinc-800", + "caret-zinc-900", + "caret-zinc-950", + "caret-neutral-50", + "caret-neutral-100", + "caret-neutral-200", + "caret-neutral-300", + "caret-neutral-400", + "caret-neutral-500", + "caret-neutral-600", + "caret-neutral-700", + "caret-neutral-800", + "caret-neutral-900", + "caret-neutral-950", + "caret-stone-50", + "caret-stone-100", + "caret-stone-200", + "caret-stone-300", + "caret-stone-400", + "caret-stone-500", + "caret-stone-600", + "caret-stone-700", + "caret-stone-800", + "caret-stone-900", + "caret-stone-950", + "caret-red-50", + "caret-red-100", + "caret-red-200", + "caret-red-300", + "caret-red-400", + "caret-red-500", + "caret-red-600", + "caret-red-700", + "caret-red-800", + "caret-red-900", + "caret-red-950", + "caret-orange-50", + "caret-orange-100", + "caret-orange-200", + "caret-orange-300", + "caret-orange-400", + "caret-orange-500", + "caret-orange-600", + "caret-orange-700", + "caret-orange-800", + "caret-orange-900", + "caret-orange-950", + "caret-amber-50", + "caret-amber-100", + "caret-amber-200", + "caret-amber-300", + "caret-amber-400", + "caret-amber-500", + "caret-amber-600", + "caret-amber-700", + "caret-amber-800", + "caret-amber-900", + "caret-amber-950", + "caret-yellow-50", + "caret-yellow-100", + "caret-yellow-200", + "caret-yellow-300", + "caret-yellow-400", + "caret-yellow-500", + "caret-yellow-600", + "caret-yellow-700", + "caret-yellow-800", + "caret-yellow-900", + "caret-yellow-950", + "caret-lime-50", + "caret-lime-100", + "caret-lime-200", + "caret-lime-300", + "caret-lime-400", + "caret-lime-500", + "caret-lime-600", + "caret-lime-700", + "caret-lime-800", + "caret-lime-900", + "caret-lime-950", + "caret-green-50", + "caret-green-100", + "caret-green-200", + "caret-green-300", + "caret-green-400", + "caret-green-500", + "caret-green-600", + "caret-green-700", + "caret-green-800", + "caret-green-900", + "caret-green-950", + "caret-emerald-50", + "caret-emerald-100", + "caret-emerald-200", + "caret-emerald-300", + "caret-emerald-400", + "caret-emerald-500", + "caret-emerald-600", + "caret-emerald-700", + "caret-emerald-800", + "caret-emerald-900", + "caret-emerald-950", + "caret-teal-50", + "caret-teal-100", + "caret-teal-200", + "caret-teal-300", + "caret-teal-400", + "caret-teal-500", + "caret-teal-600", + "caret-teal-700", + "caret-teal-800", + "caret-teal-900", + "caret-teal-950", + "caret-cyan-50", + "caret-cyan-100", + "caret-cyan-200", + "caret-cyan-300", + "caret-cyan-400", + "caret-cyan-500", + "caret-cyan-600", + "caret-cyan-700", + "caret-cyan-800", + "caret-cyan-900", + "caret-cyan-950", + "caret-sky-50", + "caret-sky-100", + "caret-sky-200", + "caret-sky-300", + "caret-sky-400", + "caret-sky-500", + "caret-sky-600", + "caret-sky-700", + "caret-sky-800", + "caret-sky-900", + "caret-sky-950", + "caret-blue-50", + "caret-blue-100", + "caret-blue-200", + "caret-blue-300", + "caret-blue-400", + "caret-blue-500", + "caret-blue-600", + "caret-blue-700", + "caret-blue-800", + "caret-blue-900", + "caret-blue-950", + "caret-indigo-50", + "caret-indigo-100", + "caret-indigo-200", + "caret-indigo-300", + "caret-indigo-400", + "caret-indigo-500", + "caret-indigo-600", + "caret-indigo-700", + "caret-indigo-800", + "caret-indigo-900", + "caret-indigo-950", + "caret-violet-50", + "caret-violet-100", + "caret-violet-200", + "caret-violet-300", + "caret-violet-400", + "caret-violet-500", + "caret-violet-600", + "caret-violet-700", + "caret-violet-800", + "caret-violet-900", + "caret-violet-950", + "caret-purple-50", + "caret-purple-100", + "caret-purple-200", + "caret-purple-300", + "caret-purple-400", + "caret-purple-500", + "caret-purple-600", + "caret-purple-700", + "caret-purple-800", + "caret-purple-900", + "caret-purple-950", + "caret-fuchsia-50", + "caret-fuchsia-100", + "caret-fuchsia-200", + "caret-fuchsia-300", + "caret-fuchsia-400", + "caret-fuchsia-500", + "caret-fuchsia-600", + "caret-fuchsia-700", + "caret-fuchsia-800", + "caret-fuchsia-900", + "caret-fuchsia-950", + "caret-pink-50", + "caret-pink-100", + "caret-pink-200", + "caret-pink-300", + "caret-pink-400", + "caret-pink-500", + "caret-pink-600", + "caret-pink-700", + "caret-pink-800", + "caret-pink-900", + "caret-pink-950", + "caret-rose-50", + "caret-rose-100", + "caret-rose-200", + "caret-rose-300", + "caret-rose-400", + "caret-rose-500", + "caret-rose-600", + "caret-rose-700", + "caret-rose-800", + "caret-rose-900", + "caret-rose-950", + "pointer-events-none", + "pointer-events-auto", + "resize-none", + "resize-y", + "resize-x", + "resize", + "scroll-auto", + "scroll-smooth", + "scroll-m-0", + "scroll-mx-0", + "scroll-my-0", + "scroll-ms-0", + "scroll-me-0", + "scroll-mt-0", + "scroll-mr-0", + "scroll-mb-0", + "scroll-ml-0", + "scroll-m-px", + "scroll-mx-px", + "scroll-my-px", + "scroll-ms-px", + "scroll-me-px", + "scroll-mt-px", + "scroll-mr-px", + "scroll-mb-px", + "scroll-ml-px", + "scroll-m-0.5", + "scroll-mx-0.5", + "scroll-my-0.5", + "scroll-ms-0.5", + "scroll-me-0.5", + "scroll-mt-0.5", + "scroll-mr-0.5", + "scroll-mb-0.5", + "scroll-ml-0.5", + "scroll-m-1", + "scroll-mx-1", + "scroll-my-1", + "scroll-ms-1", + "scroll-me-1", + "scroll-mt-1", + "scroll-mr-1", + "scroll-mb-1", + "scroll-ml-1", + "scroll-m-1.5", + "scroll-mx-1.5", + "scroll-my-1.5", + "scroll-ms-1.5", + "scroll-me-1.5", + "scroll-mt-1.5", + "scroll-mr-1.5", + "scroll-mb-1.5", + "scroll-ml-1.5", + "scroll-m-2", + "scroll-mx-2", + "scroll-my-2", + "scroll-ms-2", + "scroll-me-2", + "scroll-mt-2", + "scroll-mr-2", + "scroll-mb-2", + "scroll-ml-2", + "scroll-m-2.5", + "scroll-mx-2.5", + "scroll-my-2.5", + "scroll-ms-2.5", + "scroll-me-2.5", + "scroll-mt-2.5", + "scroll-mr-2.5", + "scroll-mb-2.5", + "scroll-ml-2.5", + "scroll-m-3", + "scroll-mx-3", + "scroll-my-3", + "scroll-ms-3", + "scroll-me-3", + "scroll-mt-3", + "scroll-mr-3", + "scroll-mb-3", + "scroll-ml-3", + "scroll-m-3.5", + "scroll-mx-3.5", + "scroll-my-3.5", + "scroll-ms-3.5", + "scroll-me-3.5", + "scroll-mt-3.5", + "scroll-mr-3.5", + "scroll-mb-3.5", + "scroll-ml-3.5", + "scroll-m-4", + "scroll-mx-4", + "scroll-my-4", + "scroll-ms-4", + "scroll-me-4", + "scroll-mt-4", + "scroll-mr-4", + "scroll-mb-4", + "scroll-ml-4", + "scroll-m-5", + "scroll-mx-5", + "scroll-my-5", + "scroll-ms-5", + "scroll-me-5", + "scroll-mt-5", + "scroll-mr-5", + "scroll-mb-5", + "scroll-ml-5", + "scroll-m-6", + "scroll-mx-6", + "scroll-my-6", + "scroll-ms-6", + "scroll-me-6", + "scroll-mt-6", + "scroll-mr-6", + "scroll-mb-6", + "scroll-ml-6", + "scroll-m-7", + "scroll-mx-7", + "scroll-my-7", + "scroll-ms-7", + "scroll-me-7", + "scroll-mt-7", + "scroll-mr-7", + "scroll-mb-7", + "scroll-ml-7", + "scroll-m-8", + "scroll-mx-8", + "scroll-my-8", + "scroll-ms-8", + "scroll-me-8", + "scroll-mt-8", + "scroll-mr-8", + "scroll-mb-8", + "scroll-ml-8", + "scroll-m-9", + "scroll-mx-9", + "scroll-my-9", + "scroll-ms-9", + "scroll-me-9", + "scroll-mt-9", + "scroll-mr-9", + "scroll-mb-9", + "scroll-ml-9", + "scroll-m-10", + "scroll-mx-10", + "scroll-my-10", + "scroll-ms-10", + "scroll-me-10", + "scroll-mt-10", + "scroll-mr-10", + "scroll-mb-10", + "scroll-ml-10", + "scroll-m-11", + "scroll-mx-11", + "scroll-my-11", + "scroll-ms-11", + "scroll-me-11", + "scroll-mt-11", + "scroll-mr-11", + "scroll-mb-11", + "scroll-ml-11", + "scroll-m-12", + "scroll-mx-12", + "scroll-my-12", + "scroll-ms-12", + "scroll-me-12", + "scroll-mt-12", + "scroll-mr-12", + "scroll-mb-12", + "scroll-ml-12", + "scroll-m-14", + "scroll-mx-14", + "scroll-my-14", + "scroll-ms-14", + "scroll-me-14", + "scroll-mt-14", + "scroll-mr-14", + "scroll-mb-14", + "scroll-ml-14", + "scroll-m-16", + "scroll-mx-16", + "scroll-my-16", + "scroll-ms-16", + "scroll-me-16", + "scroll-mt-16", + "scroll-mr-16", + "scroll-mb-16", + "scroll-ml-16", + "scroll-m-20", + "scroll-mx-20", + "scroll-my-20", + "scroll-ms-20", + "scroll-me-20", + "scroll-mt-20", + "scroll-mr-20", + "scroll-mb-20", + "scroll-ml-20", + "scroll-m-24", + "scroll-mx-24", + "scroll-my-24", + "scroll-ms-24", + "scroll-me-24", + "scroll-mt-24", + "scroll-mr-24", + "scroll-mb-24", + "scroll-ml-24", + "scroll-m-28", + "scroll-mx-28", + "scroll-my-28", + "scroll-ms-28", + "scroll-me-28", + "scroll-mt-28", + "scroll-mr-28", + "scroll-mb-28", + "scroll-ml-28", + "scroll-m-32", + "scroll-mx-32", + "scroll-my-32", + "scroll-ms-32", + "scroll-me-32", + "scroll-mt-32", + "scroll-mr-32", + "scroll-mb-32", + "scroll-ml-32", + "scroll-m-36", + "scroll-mx-36", + "scroll-my-36", + "scroll-ms-36", + "scroll-me-36", + "scroll-mt-36", + "scroll-mr-36", + "scroll-mb-36", + "scroll-ml-36", + "scroll-m-40", + "scroll-mx-40", + "scroll-my-40", + "scroll-ms-40", + "scroll-me-40", + "scroll-mt-40", + "scroll-mr-40", + "scroll-mb-40", + "scroll-ml-40", + "scroll-m-44", + "scroll-mx-44", + "scroll-my-44", + "scroll-ms-44", + "scroll-me-44", + "scroll-mt-44", + "scroll-mr-44", + "scroll-mb-44", + "scroll-ml-44", + "scroll-m-48", + "scroll-mx-48", + "scroll-my-48", + "scroll-ms-48", + "scroll-me-48", + "scroll-mt-48", + "scroll-mr-48", + "scroll-mb-48", + "scroll-ml-48", + "scroll-m-52", + "scroll-mx-52", + "scroll-my-52", + "scroll-ms-52", + "scroll-me-52", + "scroll-mt-52", + "scroll-mr-52", + "scroll-mb-52", + "scroll-ml-52", + "scroll-m-56", + "scroll-mx-56", + "scroll-my-56", + "scroll-ms-56", + "scroll-me-56", + "scroll-mt-56", + "scroll-mr-56", + "scroll-mb-56", + "scroll-ml-56", + "scroll-m-60", + "scroll-mx-60", + "scroll-my-60", + "scroll-ms-60", + "scroll-me-60", + "scroll-mt-60", + "scroll-mr-60", + "scroll-mb-60", + "scroll-ml-60", + "scroll-m-64", + "scroll-mx-64", + "scroll-my-64", + "scroll-ms-64", + "scroll-me-64", + "scroll-mt-64", + "scroll-mr-64", + "scroll-mb-64", + "scroll-ml-64", + "scroll-m-72", + "scroll-mx-72", + "scroll-my-72", + "scroll-ms-72", + "scroll-me-72", + "scroll-mt-72", + "scroll-mr-72", + "scroll-mb-72", + "scroll-ml-72", + "scroll-m-80", + "scroll-mx-80", + "scroll-my-80", + "scroll-ms-80", + "scroll-me-80", + "scroll-mt-80", + "scroll-mr-80", + "scroll-mb-80", + "scroll-ml-80", + "scroll-m-96", + "scroll-mx-96", + "scroll-my-96", + "scroll-ms-96", + "scroll-me-96", + "scroll-mt-96", + "scroll-mr-96", + "scroll-mb-96", + "scroll-ml-96", + "scroll-p-0", + "scroll-px-0", + "scroll-py-0", + "scroll-ps-0", + "scroll-pe-0", + "scroll-pt-0", + "scroll-pr-0", + "scroll-pb-0", + "scroll-pl-0", + "scroll-p-px", + "scroll-px-px", + "scroll-py-px", + "scroll-ps-px", + "scroll-pe-px", + "scroll-pt-px", + "scroll-pr-px", + "scroll-pb-px", + "scroll-pl-px", + "scroll-p-0.5", + "scroll-px-0.5", + "scroll-py-0.5", + "scroll-ps-0.5", + "scroll-pe-0.5", + "scroll-pt-0.5", + "scroll-pr-0.5", + "scroll-pb-0.5", + "scroll-pl-0.5", + "scroll-p-1", + "scroll-px-1", + "scroll-py-1", + "scroll-ps-1", + "scroll-pe-1", + "scroll-pt-1", + "scroll-pr-1", + "scroll-pb-1", + "scroll-pl-1", + "scroll-p-1.5", + "scroll-px-1.5", + "scroll-py-1.5", + "scroll-ps-1.5", + "scroll-pe-1.5", + "scroll-pt-1.5", + "scroll-pr-1.5", + "scroll-pb-1.5", + "scroll-pl-1.5", + "scroll-p-2", + "scroll-px-2", + "scroll-py-2", + "scroll-ps-2", + "scroll-pe-2", + "scroll-pt-2", + "scroll-pr-2", + "scroll-pb-2", + "scroll-pl-2", + "scroll-p-2.5", + "scroll-px-2.5", + "scroll-py-2.5", + "scroll-ps-2.5", + "scroll-pe-2.5", + "scroll-pt-2.5", + "scroll-pr-2.5", + "scroll-pb-2.5", + "scroll-pl-2.5", + "scroll-p-3", + "scroll-px-3", + "scroll-py-3", + "scroll-ps-3", + "scroll-pe-3", + "scroll-pt-3", + "scroll-pr-3", + "scroll-pb-3", + "scroll-pl-3", + "scroll-p-3.5", + "scroll-px-3.5", + "scroll-py-3.5", + "scroll-ps-3.5", + "scroll-pe-3.5", + "scroll-pt-3.5", + "scroll-pr-3.5", + "scroll-pb-3.5", + "scroll-pl-3.5", + "scroll-p-4", + "scroll-px-4", + "scroll-py-4", + "scroll-ps-4", + "scroll-pe-4", + "scroll-pt-4", + "scroll-pr-4", + "scroll-pb-4", + "scroll-pl-4", + "scroll-p-5", + "scroll-px-5", + "scroll-py-5", + "scroll-ps-5", + "scroll-pe-5", + "scroll-pt-5", + "scroll-pr-5", + "scroll-pb-5", + "scroll-pl-5", + "scroll-p-6", + "scroll-px-6", + "scroll-py-6", + "scroll-ps-6", + "scroll-pe-6", + "scroll-pt-6", + "scroll-pr-6", + "scroll-pb-6", + "scroll-pl-6", + "scroll-p-7", + "scroll-px-7", + "scroll-py-7", + "scroll-ps-7", + "scroll-pe-7", + "scroll-pt-7", + "scroll-pr-7", + "scroll-pb-7", + "scroll-pl-7", + "scroll-p-8", + "scroll-px-8", + "scroll-py-8", + "scroll-ps-8", + "scroll-pe-8", + "scroll-pt-8", + "scroll-pr-8", + "scroll-pb-8", + "scroll-pl-8", + "scroll-p-9", + "scroll-px-9", + "scroll-py-9", + "scroll-ps-9", + "scroll-pe-9", + "scroll-pt-9", + "scroll-pr-9", + "scroll-pb-9", + "scroll-pl-9", + "scroll-p-10", + "scroll-px-10", + "scroll-py-10", + "scroll-ps-10", + "scroll-pe-10", + "scroll-pt-10", + "scroll-pr-10", + "scroll-pb-10", + "scroll-pl-10", + "scroll-p-11", + "scroll-px-11", + "scroll-py-11", + "scroll-ps-11", + "scroll-pe-11", + "scroll-pt-11", + "scroll-pr-11", + "scroll-pb-11", + "scroll-pl-11", + "scroll-p-12", + "scroll-px-12", + "scroll-py-12", + "scroll-ps-12", + "scroll-pe-12", + "scroll-pt-12", + "scroll-pr-12", + "scroll-pb-12", + "scroll-pl-12", + "scroll-p-14", + "scroll-px-14", + "scroll-py-14", + "scroll-ps-14", + "scroll-pe-14", + "scroll-pt-14", + "scroll-pr-14", + "scroll-pb-14", + "scroll-pl-14", + "scroll-p-16", + "scroll-px-16", + "scroll-py-16", + "scroll-ps-16", + "scroll-pe-16", + "scroll-pt-16", + "scroll-pr-16", + "scroll-pb-16", + "scroll-pl-16", + "scroll-p-20", + "scroll-px-20", + "scroll-py-20", + "scroll-ps-20", + "scroll-pe-20", + "scroll-pt-20", + "scroll-pr-20", + "scroll-pb-20", + "scroll-pl-20", + "scroll-p-24", + "scroll-px-24", + "scroll-py-24", + "scroll-ps-24", + "scroll-pe-24", + "scroll-pt-24", + "scroll-pr-24", + "scroll-pb-24", + "scroll-pl-24", + "scroll-p-28", + "scroll-px-28", + "scroll-py-28", + "scroll-ps-28", + "scroll-pe-28", + "scroll-pt-28", + "scroll-pr-28", + "scroll-pb-28", + "scroll-pl-28", + "scroll-p-32", + "scroll-px-32", + "scroll-py-32", + "scroll-ps-32", + "scroll-pe-32", + "scroll-pt-32", + "scroll-pr-32", + "scroll-pb-32", + "scroll-pl-32", + "scroll-p-36", + "scroll-px-36", + "scroll-py-36", + "scroll-ps-36", + "scroll-pe-36", + "scroll-pt-36", + "scroll-pr-36", + "scroll-pb-36", + "scroll-pl-36", + "scroll-p-40", + "scroll-px-40", + "scroll-py-40", + "scroll-ps-40", + "scroll-pe-40", + "scroll-pt-40", + "scroll-pr-40", + "scroll-pb-40", + "scroll-pl-40", + "scroll-p-44", + "scroll-px-44", + "scroll-py-44", + "scroll-ps-44", + "scroll-pe-44", + "scroll-pt-44", + "scroll-pr-44", + "scroll-pb-44", + "scroll-pl-44", + "scroll-p-48", + "scroll-px-48", + "scroll-py-48", + "scroll-ps-48", + "scroll-pe-48", + "scroll-pt-48", + "scroll-pr-48", + "scroll-pb-48", + "scroll-pl-48", + "scroll-p-52", + "scroll-px-52", + "scroll-py-52", + "scroll-ps-52", + "scroll-pe-52", + "scroll-pt-52", + "scroll-pr-52", + "scroll-pb-52", + "scroll-pl-52", + "scroll-p-56", + "scroll-px-56", + "scroll-py-56", + "scroll-ps-56", + "scroll-pe-56", + "scroll-pt-56", + "scroll-pr-56", + "scroll-pb-56", + "scroll-pl-56", + "scroll-p-60", + "scroll-px-60", + "scroll-py-60", + "scroll-ps-60", + "scroll-pe-60", + "scroll-pt-60", + "scroll-pr-60", + "scroll-pb-60", + "scroll-pl-60", + "scroll-p-64", + "scroll-px-64", + "scroll-py-64", + "scroll-ps-64", + "scroll-pe-64", + "scroll-pt-64", + "scroll-pr-64", + "scroll-pb-64", + "scroll-pl-64", + "scroll-p-72", + "scroll-px-72", + "scroll-py-72", + "scroll-ps-72", + "scroll-pe-72", + "scroll-pt-72", + "scroll-pr-72", + "scroll-pb-72", + "scroll-pl-72", + "scroll-p-80", + "scroll-px-80", + "scroll-py-80", + "scroll-ps-80", + "scroll-pe-80", + "scroll-pt-80", + "scroll-pr-80", + "scroll-pb-80", + "scroll-pl-80", + "scroll-p-96", + "scroll-px-96", + "scroll-py-96", + "scroll-ps-96", + "scroll-pe-96", + "scroll-pt-96", + "scroll-pr-96", + "scroll-pb-96", + "scroll-pl-96", + "snap-start", + "snap-end", + "snap-center", + "snap-align-none", + "snap-normal", + "snap-always", + "snap-none", + "snap-x", + "snap-y", + "snap-both", + "snap-mandatory", + "snap-proximity", + "touch-auto", + "touch-none", + "touch-pan-x", + "touch-pan-left", + "touch-pan-right", + "touch-pan-y", + "touch-pan-up", + "touch-pan-down", + "touch-pinch-zoom", + "touch-manipulation", + "select-none", + "select-text", + "select-all", + "select-auto", + "will-change-auto", + "will-change-scroll", + "will-change-contents", + "will-change-transform", + "fill-none", + "fill-inherit", + "fill-current", + "fill-transparent", + "fill-black", + "fill-white", + "fill-slate-50", + "fill-slate-100", + "fill-slate-200", + "fill-slate-300", + "fill-slate-400", + "fill-slate-500", + "fill-slate-600", + "fill-slate-700", + "fill-slate-800", + "fill-slate-900", + "fill-slate-950", + "fill-gray-50", + "fill-gray-100", + "fill-gray-200", + "fill-gray-300", + "fill-gray-400", + "fill-gray-500", + "fill-gray-600", + "fill-gray-700", + "fill-gray-800", + "fill-gray-900", + "fill-gray-950", + "fill-zinc-50", + "fill-zinc-100", + "fill-zinc-200", + "fill-zinc-300", + "fill-zinc-400", + "fill-zinc-500", + "fill-zinc-600", + "fill-zinc-700", + "fill-zinc-800", + "fill-zinc-900", + "fill-zinc-950", + "fill-neutral-50", + "fill-neutral-100", + "fill-neutral-200", + "fill-neutral-300", + "fill-neutral-400", + "fill-neutral-500", + "fill-neutral-600", + "fill-neutral-700", + "fill-neutral-800", + "fill-neutral-900", + "fill-neutral-950", + "fill-stone-50", + "fill-stone-100", + "fill-stone-200", + "fill-stone-300", + "fill-stone-400", + "fill-stone-500", + "fill-stone-600", + "fill-stone-700", + "fill-stone-800", + "fill-stone-900", + "fill-stone-950", + "fill-red-50", + "fill-red-100", + "fill-red-200", + "fill-red-300", + "fill-red-400", + "fill-red-500", + "fill-red-600", + "fill-red-700", + "fill-red-800", + "fill-red-900", + "fill-red-950", + "fill-orange-50", + "fill-orange-100", + "fill-orange-200", + "fill-orange-300", + "fill-orange-400", + "fill-orange-500", + "fill-orange-600", + "fill-orange-700", + "fill-orange-800", + "fill-orange-900", + "fill-orange-950", + "fill-amber-50", + "fill-amber-100", + "fill-amber-200", + "fill-amber-300", + "fill-amber-400", + "fill-amber-500", + "fill-amber-600", + "fill-amber-700", + "fill-amber-800", + "fill-amber-900", + "fill-amber-950", + "fill-yellow-50", + "fill-yellow-100", + "fill-yellow-200", + "fill-yellow-300", + "fill-yellow-400", + "fill-yellow-500", + "fill-yellow-600", + "fill-yellow-700", + "fill-yellow-800", + "fill-yellow-900", + "fill-yellow-950", + "fill-lime-50", + "fill-lime-100", + "fill-lime-200", + "fill-lime-300", + "fill-lime-400", + "fill-lime-500", + "fill-lime-600", + "fill-lime-700", + "fill-lime-800", + "fill-lime-900", + "fill-lime-950", + "fill-green-50", + "fill-green-100", + "fill-green-200", + "fill-green-300", + "fill-green-400", + "fill-green-500", + "fill-green-600", + "fill-green-700", + "fill-green-800", + "fill-green-900", + "fill-green-950", + "fill-emerald-50", + "fill-emerald-100", + "fill-emerald-200", + "fill-emerald-300", + "fill-emerald-400", + "fill-emerald-500", + "fill-emerald-600", + "fill-emerald-700", + "fill-emerald-800", + "fill-emerald-900", + "fill-emerald-950", + "fill-teal-50", + "fill-teal-100", + "fill-teal-200", + "fill-teal-300", + "fill-teal-400", + "fill-teal-500", + "fill-teal-600", + "fill-teal-700", + "fill-teal-800", + "fill-teal-900", + "fill-teal-950", + "fill-cyan-50", + "fill-cyan-100", + "fill-cyan-200", + "fill-cyan-300", + "fill-cyan-400", + "fill-cyan-500", + "fill-cyan-600", + "fill-cyan-700", + "fill-cyan-800", + "fill-cyan-900", + "fill-cyan-950", + "fill-sky-50", + "fill-sky-100", + "fill-sky-200", + "fill-sky-300", + "fill-sky-400", + "fill-sky-500", + "fill-sky-600", + "fill-sky-700", + "fill-sky-800", + "fill-sky-900", + "fill-sky-950", + "fill-blue-50", + "fill-blue-100", + "fill-blue-200", + "fill-blue-300", + "fill-blue-400", + "fill-blue-500", + "fill-blue-600", + "fill-blue-700", + "fill-blue-800", + "fill-blue-900", + "fill-blue-950", + "fill-indigo-50", + "fill-indigo-100", + "fill-indigo-200", + "fill-indigo-300", + "fill-indigo-400", + "fill-indigo-500", + "fill-indigo-600", + "fill-indigo-700", + "fill-indigo-800", + "fill-indigo-900", + "fill-indigo-950", + "fill-violet-50", + "fill-violet-100", + "fill-violet-200", + "fill-violet-300", + "fill-violet-400", + "fill-violet-500", + "fill-violet-600", + "fill-violet-700", + "fill-violet-800", + "fill-violet-900", + "fill-violet-950", + "fill-purple-50", + "fill-purple-100", + "fill-purple-200", + "fill-purple-300", + "fill-purple-400", + "fill-purple-500", + "fill-purple-600", + "fill-purple-700", + "fill-purple-800", + "fill-purple-900", + "fill-purple-950", + "fill-fuchsia-50", + "fill-fuchsia-100", + "fill-fuchsia-200", + "fill-fuchsia-300", + "fill-fuchsia-400", + "fill-fuchsia-500", + "fill-fuchsia-600", + "fill-fuchsia-700", + "fill-fuchsia-800", + "fill-fuchsia-900", + "fill-fuchsia-950", + "fill-pink-50", + "fill-pink-100", + "fill-pink-200", + "fill-pink-300", + "fill-pink-400", + "fill-pink-500", + "fill-pink-600", + "fill-pink-700", + "fill-pink-800", + "fill-pink-900", + "fill-pink-950", + "fill-rose-50", + "fill-rose-100", + "fill-rose-200", + "fill-rose-300", + "fill-rose-400", + "fill-rose-500", + "fill-rose-600", + "fill-rose-700", + "fill-rose-800", + "fill-rose-900", + "fill-rose-950", + "stroke-none", + "stroke-inherit", + "stroke-current", + "stroke-transparent", + "stroke-black", + "stroke-white", + "stroke-slate-50", + "stroke-slate-100", + "stroke-slate-200", + "stroke-slate-300", + "stroke-slate-400", + "stroke-slate-500", + "stroke-slate-600", + "stroke-slate-700", + "stroke-slate-800", + "stroke-slate-900", + "stroke-slate-950", + "stroke-gray-50", + "stroke-gray-100", + "stroke-gray-200", + "stroke-gray-300", + "stroke-gray-400", + "stroke-gray-500", + "stroke-gray-600", + "stroke-gray-700", + "stroke-gray-800", + "stroke-gray-900", + "stroke-gray-950", + "stroke-zinc-50", + "stroke-zinc-100", + "stroke-zinc-200", + "stroke-zinc-300", + "stroke-zinc-400", + "stroke-zinc-500", + "stroke-zinc-600", + "stroke-zinc-700", + "stroke-zinc-800", + "stroke-zinc-900", + "stroke-zinc-950", + "stroke-neutral-50", + "stroke-neutral-100", + "stroke-neutral-200", + "stroke-neutral-300", + "stroke-neutral-400", + "stroke-neutral-500", + "stroke-neutral-600", + "stroke-neutral-700", + "stroke-neutral-800", + "stroke-neutral-900", + "stroke-neutral-950", + "stroke-stone-50", + "stroke-stone-100", + "stroke-stone-200", + "stroke-stone-300", + "stroke-stone-400", + "stroke-stone-500", + "stroke-stone-600", + "stroke-stone-700", + "stroke-stone-800", + "stroke-stone-900", + "stroke-stone-950", + "stroke-red-50", + "stroke-red-100", + "stroke-red-200", + "stroke-red-300", + "stroke-red-400", + "stroke-red-500", + "stroke-red-600", + "stroke-red-700", + "stroke-red-800", + "stroke-red-900", + "stroke-red-950", + "stroke-orange-50", + "stroke-orange-100", + "stroke-orange-200", + "stroke-orange-300", + "stroke-orange-400", + "stroke-orange-500", + "stroke-orange-600", + "stroke-orange-700", + "stroke-orange-800", + "stroke-orange-900", + "stroke-orange-950", + "stroke-amber-50", + "stroke-amber-100", + "stroke-amber-200", + "stroke-amber-300", + "stroke-amber-400", + "stroke-amber-500", + "stroke-amber-600", + "stroke-amber-700", + "stroke-amber-800", + "stroke-amber-900", + "stroke-amber-950", + "stroke-yellow-50", + "stroke-yellow-100", + "stroke-yellow-200", + "stroke-yellow-300", + "stroke-yellow-400", + "stroke-yellow-500", + "stroke-yellow-600", + "stroke-yellow-700", + "stroke-yellow-800", + "stroke-yellow-900", + "stroke-yellow-950", + "stroke-lime-50", + "stroke-lime-100", + "stroke-lime-200", + "stroke-lime-300", + "stroke-lime-400", + "stroke-lime-500", + "stroke-lime-600", + "stroke-lime-700", + "stroke-lime-800", + "stroke-lime-900", + "stroke-lime-950", + "stroke-green-50", + "stroke-green-100", + "stroke-green-200", + "stroke-green-300", + "stroke-green-400", + "stroke-green-500", + "stroke-green-600", + "stroke-green-700", + "stroke-green-800", + "stroke-green-900", + "stroke-green-950", + "stroke-emerald-50", + "stroke-emerald-100", + "stroke-emerald-200", + "stroke-emerald-300", + "stroke-emerald-400", + "stroke-emerald-500", + "stroke-emerald-600", + "stroke-emerald-700", + "stroke-emerald-800", + "stroke-emerald-900", + "stroke-emerald-950", + "stroke-teal-50", + "stroke-teal-100", + "stroke-teal-200", + "stroke-teal-300", + "stroke-teal-400", + "stroke-teal-500", + "stroke-teal-600", + "stroke-teal-700", + "stroke-teal-800", + "stroke-teal-900", + "stroke-teal-950", + "stroke-cyan-50", + "stroke-cyan-100", + "stroke-cyan-200", + "stroke-cyan-300", + "stroke-cyan-400", + "stroke-cyan-500", + "stroke-cyan-600", + "stroke-cyan-700", + "stroke-cyan-800", + "stroke-cyan-900", + "stroke-cyan-950", + "stroke-sky-50", + "stroke-sky-100", + "stroke-sky-200", + "stroke-sky-300", + "stroke-sky-400", + "stroke-sky-500", + "stroke-sky-600", + "stroke-sky-700", + "stroke-sky-800", + "stroke-sky-900", + "stroke-sky-950", + "stroke-blue-50", + "stroke-blue-100", + "stroke-blue-200", + "stroke-blue-300", + "stroke-blue-400", + "stroke-blue-500", + "stroke-blue-600", + "stroke-blue-700", + "stroke-blue-800", + "stroke-blue-900", + "stroke-blue-950", + "stroke-indigo-50", + "stroke-indigo-100", + "stroke-indigo-200", + "stroke-indigo-300", + "stroke-indigo-400", + "stroke-indigo-500", + "stroke-indigo-600", + "stroke-indigo-700", + "stroke-indigo-800", + "stroke-indigo-900", + "stroke-indigo-950", + "stroke-violet-50", + "stroke-violet-100", + "stroke-violet-200", + "stroke-violet-300", + "stroke-violet-400", + "stroke-violet-500", + "stroke-violet-600", + "stroke-violet-700", + "stroke-violet-800", + "stroke-violet-900", + "stroke-violet-950", + "stroke-purple-50", + "stroke-purple-100", + "stroke-purple-200", + "stroke-purple-300", + "stroke-purple-400", + "stroke-purple-500", + "stroke-purple-600", + "stroke-purple-700", + "stroke-purple-800", + "stroke-purple-900", + "stroke-purple-950", + "stroke-fuchsia-50", + "stroke-fuchsia-100", + "stroke-fuchsia-200", + "stroke-fuchsia-300", + "stroke-fuchsia-400", + "stroke-fuchsia-500", + "stroke-fuchsia-600", + "stroke-fuchsia-700", + "stroke-fuchsia-800", + "stroke-fuchsia-900", + "stroke-fuchsia-950", + "stroke-pink-50", + "stroke-pink-100", + "stroke-pink-200", + "stroke-pink-300", + "stroke-pink-400", + "stroke-pink-500", + "stroke-pink-600", + "stroke-pink-700", + "stroke-pink-800", + "stroke-pink-900", + "stroke-pink-950", + "stroke-rose-50", + "stroke-rose-100", + "stroke-rose-200", + "stroke-rose-300", + "stroke-rose-400", + "stroke-rose-500", + "stroke-rose-600", + "stroke-rose-700", + "stroke-rose-800", + "stroke-rose-900", + "stroke-rose-950", + "stroke-0", + "stroke-1", + "stroke-2", + "sr-only", + "not-sr-only", + "forced-color-adjust-auto", + "forced-color-adjust-none", + "transform", + "transform-none", + "transform-gpu", + "transform-cpu", + "group", + "peer", ]); diff --git a/tests/container-runtime-regressions.test.cjs b/tests/container-runtime-regressions.test.cjs index e4b259f..602a9d8 100644 --- a/tests/container-runtime-regressions.test.cjs +++ b/tests/container-runtime-regressions.test.cjs @@ -1,17 +1,17 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const path = require('node:path'); +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); -const projectRoot = path.resolve(__dirname, '..'); +const projectRoot = path.resolve(__dirname, ".."); function readProjectFile(relativePath) { - return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8'); + return fs.readFileSync(path.join(projectRoot, relativePath), "utf8"); } -test('browser launchers include container-safe Chrome sandbox flags', () => { - const mainPuppeteer = readProjectFile('src/services/puppeteer.ts'); - const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts'); +test("browser launchers include container-safe Chrome sandbox flags", () => { + const mainPuppeteer = readProjectFile("src/services/puppeteer.ts"); + const serpPuppeteer = readProjectFile("src/services/serp/puppeteer.ts"); for (const source of [mainPuppeteer, serpPuppeteer]) { assert.match(source, /pipe:\s*true/); @@ -20,85 +20,123 @@ test('browser launchers include container-safe Chrome sandbox flags', () => { } }); -test('SERP puppeteer control uses an absolute control timeout instead of a fixed post-navigation race', () => { - const serpPuppeteer = readProjectFile('src/services/serp/puppeteer.ts'); +test("SERP puppeteer control uses an absolute control timeout instead of a fixed post-navigation race", () => { + const serpPuppeteer = readProjectFile("src/services/serp/puppeteer.ts"); assert.match(serpPuppeteer, /waitUntil:\s*\['domcontentloaded', 'load'\]/); - assert.match(serpPuppeteer, /setTimeout\(\(\) => \{\s*resultDeferred\.reject\(new TimeoutError/); + assert.match( + serpPuppeteer, + /setTimeout\(\(\) => \{\s*resultDeferred\.reject\(new TimeoutError/, + ); assert.doesNotMatch(serpPuppeteer, /networkidle0/); assert.doesNotMatch(serpPuppeteer, /await delay\(5000\)/); }); -test('canvas service falls back when bundled font asset is missing', () => { - const canvasService = readProjectFile('src/services/canvas.ts'); +test("canvas service falls back when bundled font asset is missing", () => { + const canvasService = readProjectFile("src/services/canvas.ts"); assert.match(canvasService, /ENOENT/); assert.match(canvasService, /Bundled font asset unavailable/); }); -test('search host includes Google scraping fallback for standalone builds', () => { - const searcherHost = readProjectFile('src/api/searcher.ts'); +test("search host includes Google scraping fallback for standalone builds", () => { + const searcherHost = readProjectFile("src/api/searcher.ts"); assert.match(searcherHost, /protected googleSerp: GoogleSERP/); - assert.match(searcherHost, /protected standaloneFallback: StandaloneSearchFallbackService/); + assert.match( + searcherHost, + /protected standaloneFallback: StandaloneSearchFallbackService/, + ); assert.match(searcherHost, /yield this\.googleSerp/); assert.match(searcherHost, /yield this\.standaloneFallback/); - assert.match(searcherHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/); + assert.match( + searcherHost, + /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/, + ); }); -test('SERP host skips Serper providers when no API key is configured', () => { - const serpHost = readProjectFile('src/api/serp.ts'); +test("SERP host skips Serper providers when no API key is configured", () => { + const serpHost = readProjectFile("src/api/serp.ts"); - assert.match(serpHost, /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/); + assert.match( + serpHost, + /const includeSerper = Boolean\(this\.secretExposer\.SERPER_SEARCH_API_KEY\)/, + ); assert.match(serpHost, /yield this\.standaloneFallback/); }); -test('standalone fallback search provider is available for local web search without private APIs', () => { - const fallbackProvider = readProjectFile('src/services/serp/standalone-fallback.ts'); +test("standalone fallback search provider is available for local web search without private APIs", () => { + const fallbackProvider = readProjectFile( + "src/services/serp/standalone-fallback.ts", + ); assert.match(fallbackProvider, /DuckDuckGo HTML fallback/i); - assert.match(fallbackProvider, /querySelectorAll\('\.result:not\(\.result--ad\)'\)/); + assert.match( + fallbackProvider, + /querySelectorAll\('\.result:not\(\.result--ad\)'\)/, + ); assert.match(fallbackProvider, /ad_domain/); }); -test('google search provider can fall back to direct requests when no proxy is configured', () => { - const googleSerp = readProjectFile('src/services/serp/google.ts'); +test("google search provider can fall back to direct requests when no proxy is configured", () => { + const googleSerp = readProjectFile("src/services/serp/google.ts"); assert.match(googleSerp, /falling back to direct Google request/i); }); -test('standalone URL validation can allow Docker DNS rewrite ranges behind an explicit env gate', () => { - const miscService = readProjectFile('src/services/misc.ts'); +test("standalone URL validation can allow Docker DNS rewrite ranges behind an explicit env gate", () => { + const miscService = readProjectFile("src/services/misc.ts"); assert.match(miscService, /STANDALONE_ALLOW_INTERNAL_DNS_REWRITE/); assert.match(miscService, /198\.18/); assert.match(miscService, /fd00::/i); }); -test('standalone services use extended boot timeouts for cold starts', () => { - const bootTimeouts = readProjectFile('src/services/boot-timeouts.ts'); - const crawlHost = readProjectFile('src/api/crawler.ts'); - const searcherHost = readProjectFile('src/api/searcher.ts'); - const serpHost = readProjectFile('src/api/serp.ts'); - const crawlServer = readProjectFile('src/stand-alone/crawl.ts'); - const searchServer = readProjectFile('src/stand-alone/search.ts'); - const serpServer = readProjectFile('src/stand-alone/serp.ts'); - const googleSerp = readProjectFile('src/services/serp/google.ts'); +test("standalone services use extended boot timeouts for cold starts", () => { + const bootTimeouts = readProjectFile("src/services/boot-timeouts.ts"); + const crawlHost = readProjectFile("src/api/crawler.ts"); + const searcherHost = readProjectFile("src/api/searcher.ts"); + const serpHost = readProjectFile("src/api/serp.ts"); + const crawlServer = readProjectFile("src/stand-alone/crawl.ts"); + const searchServer = readProjectFile("src/stand-alone/search.ts"); + const serpServer = readProjectFile("src/stand-alone/serp.ts"); + const googleSerp = readProjectFile("src/services/serp/google.ts"); assert.match(bootTimeouts, /STANDALONE_BOOT_TIMEOUT_MS = 90_000/); assert.match(bootTimeouts, /SERP_BOOT_TIMEOUT_MS = 90_000/); assert.match(crawlHost, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); - assert.match(crawlHost, /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match( + crawlHost, + /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/, + ); assert.match(searcherHost, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); - assert.match(searcherHost, /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match( + searcherHost, + /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/, + ); assert.match(serpHost, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); - assert.match(serpHost, /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match( + serpHost, + /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/, + ); assert.match(crawlServer, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); - assert.match(crawlServer, /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match( + crawlServer, + /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/, + ); assert.match(searchServer, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); - assert.match(searchServer, /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match( + searchServer, + /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/, + ); assert.match(serpServer, /dependencyReady\(STANDALONE_BOOT_TIMEOUT_MS\)/); - assert.match(serpServer, /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/); + assert.match( + serpServer, + /override dependencyReady\(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS\)/, + ); assert.match(googleSerp, /dependencyReady\(SERP_BOOT_TIMEOUT_MS\)/); - assert.match(googleSerp, /override dependencyReady\(timeoutMilliseconds = SERP_BOOT_TIMEOUT_MS\)/); + assert.match( + googleSerp, + /override dependencyReady\(timeoutMilliseconds = SERP_BOOT_TIMEOUT_MS\)/, + ); }); diff --git a/tests/github-automation.test.cjs b/tests/github-automation.test.cjs index 52152bf..f6e0e02 100644 --- a/tests/github-automation.test.cjs +++ b/tests/github-automation.test.cjs @@ -1,56 +1,65 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const path = require('node:path'); +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); -const projectRoot = path.resolve(__dirname, '..'); +const projectRoot = path.resolve(__dirname, ".."); function read(relativePath) { - return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8'); + return fs.readFileSync(path.join(projectRoot, relativePath), "utf8"); } -test('legacy Cloud Run deployment workflow has been removed', () => { - const legacyWorkflow = path.join(projectRoot, '.github', 'workflows', 'cd.yml'); +test("legacy Cloud Run deployment workflow has been removed", () => { + const legacyWorkflow = path.join( + projectRoot, + ".github", + "workflows", + "cd.yml", + ); assert.equal(fs.existsSync(legacyWorkflow), false); }); -test('repository automation files exist for CI, container publish, and Dependabot', () => { +test("repository automation files exist for CI, container publish, and Dependabot", () => { const expectedFiles = [ - 'eslint.config.cjs', - '.github/workflows/ci.yml', - '.github/workflows/codeql.yml', - '.github/workflows/image.yml', - '.github/workflows/dependabot-auto-merge.yml', - '.github/dependabot.yml', + "eslint.config.cjs", + ".github/workflows/ci.yml", + ".github/workflows/codeql.yml", + ".github/workflows/image.yml", + ".github/workflows/dependabot-auto-merge.yml", + ".github/dependabot.yml", ]; for (const relativePath of expectedFiles) { - assert.equal(fs.existsSync(path.join(projectRoot, relativePath)), true, `${relativePath} should exist`); + assert.equal( + fs.existsSync(path.join(projectRoot, relativePath)), + true, + `${relativePath} should exist`, + ); } }); -test('container image workflow publishes to GHCR instead of legacy GCP registries', () => { - const workflow = read('.github/workflows/image.yml'); +test("container image workflow publishes to GHCR instead of legacy GCP registries", () => { + const workflow = read(".github/workflows/image.yml"); assert.match(workflow, /ghcr\.io/); assert.doesNotMatch(workflow, /gcloud|us-docker\.pkg\.dev/); }); -test('container image publishing waits for CI instead of running on pull requests directly', () => { - const workflow = read('.github/workflows/image.yml'); +test("container image publishing waits for CI instead of running on pull requests directly", () => { + const workflow = read(".github/workflows/image.yml"); assert.match(workflow, /push:\s*\n\s*branches:\s*\n\s*- main/); assert.doesNotMatch(workflow, /pull_request:/); assert.doesNotMatch(workflow, /workflow_run:/); }); -test('workflows opt into Node 24 for JavaScript-based GitHub Actions', () => { +test("workflows opt into Node 24 for JavaScript-based GitHub Actions", () => { const workflows = [ - read('.github/workflows/ci.yml'), - read('.github/workflows/codeql.yml'), - read('.github/workflows/image.yml'), - read('.github/workflows/dependabot-auto-merge.yml'), + read(".github/workflows/ci.yml"), + read(".github/workflows/codeql.yml"), + read(".github/workflows/image.yml"), + read(".github/workflows/dependabot-auto-merge.yml"), ]; for (const workflow of workflows) { @@ -58,8 +67,8 @@ test('workflows opt into Node 24 for JavaScript-based GitHub Actions', () => { } }); -test('CodeQL workflow uses advanced setup with repository-managed configuration', () => { - const workflow = read('.github/workflows/codeql.yml'); +test("CodeQL workflow uses advanced setup with repository-managed configuration", () => { + const workflow = read(".github/workflows/codeql.yml"); assert.match(workflow, /uses:\s*actions\/checkout@v\d+/); assert.match(workflow, /uses:\s*github\/codeql-action\/init@v4/); @@ -68,16 +77,16 @@ test('CodeQL workflow uses advanced setup with repository-managed configuration' assert.match(workflow, /language:\s*javascript-typescript/); }); -test('dependabot config covers npm, docker, and github-actions updates', () => { - const dependabot = read('.github/dependabot.yml'); +test("dependabot config covers npm, docker, and github-actions updates", () => { + const dependabot = read(".github/dependabot.yml"); assert.match(dependabot, /package-ecosystem: npm/); assert.match(dependabot, /package-ecosystem: docker/); assert.match(dependabot, /package-ecosystem: github-actions/); }); -test('dependabot groups include major upgrades instead of suppressing them', () => { - const dependabot = read('.github/dependabot.yml'); +test("dependabot groups include major upgrades instead of suppressing them", () => { + const dependabot = read(".github/dependabot.yml"); assert.match(dependabot, /production-dependencies:[\s\S]*-\s+major/); assert.match(dependabot, /development-dependencies:[\s\S]*-\s+major/); @@ -88,24 +97,24 @@ test('dependabot groups include major upgrades instead of suppressing them', () assert.doesNotMatch(dependabot, /dependency-name:\s*node/); }); -test('CI workflow runs lint before tests and build', () => { - const workflow = read('.github/workflows/ci.yml'); +test("CI workflow runs lint before tests and build", () => { + const workflow = read(".github/workflows/ci.yml"); assert.match(workflow, /run: npm run lint/); assert.match(workflow, /run: npm run test:ci/); assert.match(workflow, /run: npm run build/); }); -test('CI validates Docker builds for pull requests without publishing images', () => { - const workflow = read('.github/workflows/ci.yml'); +test("CI validates Docker builds for pull requests without publishing images", () => { + const workflow = read(".github/workflows/ci.yml"); assert.match(workflow, /name: Validate container build/); assert.match(workflow, /if: github\.event_name == 'pull_request'/); assert.match(workflow, /push: false/); }); -test('dependabot auto-merge workflow updates stale branches before enabling auto-merge', () => { - const workflow = read('.github/workflows/dependabot-auto-merge.yml'); +test("dependabot auto-merge workflow updates stale branches before enabling auto-merge", () => { + const workflow = read(".github/workflows/dependabot-auto-merge.yml"); assert.match(workflow, /gh pr update-branch/); assert.match(workflow, /mergeStateStatus/); @@ -114,8 +123,8 @@ test('dependabot auto-merge workflow updates stale branches before enabling auto assert.match(workflow, /gh pr list/); }); -test('Dockerfile is self-contained and no longer relies on curl-impersonate', () => { - const dockerfile = read('Dockerfile'); +test("Dockerfile is self-contained and no longer relies on curl-impersonate", () => { + const dockerfile = read("Dockerfile"); assert.match(dockerfile, /FROM node:22-bookworm-slim/); assert.match(dockerfile, /PUPPETEER_SKIP_DOWNLOAD=true/); diff --git a/tests/integrity-check.test.cjs b/tests/integrity-check.test.cjs index 241137d..9cae46d 100644 --- a/tests/integrity-check.test.cjs +++ b/tests/integrity-check.test.cjs @@ -1,19 +1,22 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const os = require('node:os'); -const path = require('node:path'); -const { spawnSync } = require('node:child_process'); +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); -test('integrity check warns instead of failing when GeoLite asset is missing', () => { - const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'xread-integrity-')); - const scriptPath = path.join(tempRoot, 'integrity-check.cjs'); +test("integrity check warns instead of failing when GeoLite asset is missing", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "xread-integrity-")); + const scriptPath = path.join(tempRoot, "integrity-check.cjs"); - fs.copyFileSync(path.resolve(__dirname, '..', 'integrity-check.cjs'), scriptPath); + fs.copyFileSync( + path.resolve(__dirname, "..", "integrity-check.cjs"), + scriptPath, + ); const result = spawnSync(process.execPath, [scriptPath], { cwd: tempRoot, - encoding: 'utf8', + encoding: "utf8", }); assert.equal(result.status, 0); diff --git a/tests/open-source-build.test.cjs b/tests/open-source-build.test.cjs index c296910..4bcd73f 100644 --- a/tests/open-source-build.test.cjs +++ b/tests/open-source-build.test.cjs @@ -1,33 +1,41 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const path = require('node:path'); -const { spawnSync } = require('node:child_process'); +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); -const projectRoot = path.resolve(__dirname, '..'); +const projectRoot = path.resolve(__dirname, ".."); -test('src/shared is a real directory in the repository', () => { - const sharedPath = path.join(projectRoot, 'src', 'shared'); +test("src/shared is a real directory in the repository", () => { + const sharedPath = path.join(projectRoot, "src", "shared"); const stat = fs.statSync(sharedPath); assert.equal(stat.isDirectory(), true); }); -test('default build script uses standard TypeScript compiler', () => { - const packageJson = JSON.parse(fs.readFileSync(path.join(projectRoot, 'package.json'), 'utf8')); +test("default build script uses standard TypeScript compiler", () => { + const packageJson = JSON.parse( + fs.readFileSync(path.join(projectRoot, "package.json"), "utf8"), + ); assert.match(packageJson.scripts.build, /\btsc\b/); assert.doesNotMatch(packageJson.scripts.build, /scripts\/transpile\.cjs/); }); -test('project type-check build passes with tsc', () => { - const tscJsPath = path.join(projectRoot, 'node_modules', 'typescript', 'lib', 'tsc.js'); +test("project type-check build passes with tsc", () => { + const tscJsPath = path.join( + projectRoot, + "node_modules", + "typescript", + "lib", + "tsc.js", + ); const result = spawnSync( process.execPath, - [tscJsPath, '-p', '.', '--pretty', 'false'], + [tscJsPath, "-p", ".", "--pretty", "false"], { cwd: projectRoot, - encoding: 'utf8', + encoding: "utf8", shell: false, }, ); diff --git a/tests/prepare-licensed-assets.test.cjs b/tests/prepare-licensed-assets.test.cjs index d0b8b0b..b70dab7 100644 --- a/tests/prepare-licensed-assets.test.cjs +++ b/tests/prepare-licensed-assets.test.cjs @@ -1,28 +1,31 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const os = require('node:os'); -const path = require('node:path'); +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); -test('ensureLicensedAssets creates GeoLite database when missing', async () => { - const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'xread-licensed-')); - const targetFile = path.join(tempRoot, 'licensed', 'GeoLite2-City.mmdb'); +test("ensureLicensedAssets creates GeoLite database when missing", async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "xread-licensed-")); + const targetFile = path.join(tempRoot, "licensed", "GeoLite2-City.mmdb"); let downloadCalls = 0; - const { ensureLicensedAssets, DEFAULT_ASSETS } = require('../scripts/prepare-licensed-assets.cjs'); + const { + ensureLicensedAssets, + DEFAULT_ASSETS, + } = require("../scripts/prepare-licensed-assets.cjs"); await ensureLicensedAssets({ rootDir: tempRoot, assets: DEFAULT_ASSETS, downloadAsset: async (asset, destination) => { downloadCalls += 1; - assert.equal(asset.path, 'licensed/GeoLite2-City.mmdb'); + assert.equal(asset.path, "licensed/GeoLite2-City.mmdb"); fs.mkdirSync(path.dirname(destination), { recursive: true }); - fs.writeFileSync(destination, Buffer.from('fake-mmdb')); + fs.writeFileSync(destination, Buffer.from("fake-mmdb")); }, }); assert.equal(downloadCalls, 1); assert.equal(fs.existsSync(targetFile), true); - assert.equal(fs.readFileSync(targetFile, 'utf8'), 'fake-mmdb'); + assert.equal(fs.readFileSync(targetFile, "utf8"), "fake-mmdb"); }); diff --git a/tests/security-governance.test.cjs b/tests/security-governance.test.cjs index f635116..bcd8ad9 100644 --- a/tests/security-governance.test.cjs +++ b/tests/security-governance.test.cjs @@ -1,30 +1,36 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); -const fs = require('node:fs'); -const path = require('node:path'); +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); -const projectRoot = path.resolve(__dirname, '..'); +const projectRoot = path.resolve(__dirname, ".."); function read(relativePath) { - return fs.readFileSync(path.join(projectRoot, relativePath), 'utf8'); + return fs.readFileSync(path.join(projectRoot, relativePath), "utf8"); } -test('package.json exposes the security audit script and excludes removed cloud dependencies', () => { - const packageJson = JSON.parse(read('package.json')); +test("package.json exposes the security audit script and excludes removed cloud dependencies", () => { + const packageJson = JSON.parse(read("package.json")); - assert.equal(packageJson.scripts['security:audit'], 'node ./scripts/security-audit-report.cjs'); - assert.match(packageJson.scripts['test:ci'], /tests\/security-governance\.test\.cjs/); - assert.ok(!('@google-cloud/translate' in packageJson.dependencies)); - assert.ok(!('express' in packageJson.dependencies)); - assert.ok(!('firebase-admin' in packageJson.dependencies)); - assert.ok(!('firebase-functions' in packageJson.dependencies)); - assert.ok(!('firebase-functions-test' in packageJson.devDependencies)); - assert.ok(!('openai' in packageJson.dependencies)); - assert.ok(!('replicate' in packageJson.devDependencies)); + assert.equal( + packageJson.scripts["security:audit"], + "node ./scripts/security-audit-report.cjs", + ); + assert.match( + packageJson.scripts["test:ci"], + /tests\/security-governance\.test\.cjs/, + ); + assert.ok(!("@google-cloud/translate" in packageJson.dependencies)); + assert.ok(!("express" in packageJson.dependencies)); + assert.ok(!("firebase-admin" in packageJson.dependencies)); + assert.ok(!("firebase-functions" in packageJson.dependencies)); + assert.ok(!("firebase-functions-test" in packageJson.devDependencies)); + assert.ok(!("openai" in packageJson.dependencies)); + assert.ok(!("replicate" in packageJson.devDependencies)); }); -test('security governance workflow enforces npm audit policy and uploads reports', () => { - const workflow = read('.github/workflows/security-governance.yml'); +test("security governance workflow enforces npm audit policy and uploads reports", () => { + const workflow = read(".github/workflows/security-governance.yml"); assert.match(workflow, /name:\s+Security Governance/); assert.match(workflow, /run:\s+npm run security:audit/); @@ -32,8 +38,8 @@ test('security governance workflow enforces npm audit policy and uploads reports assert.match(workflow, /cron:\s+'31 17 \* \* 1'/); }); -test('security baseline file is present and ready for future exceptions', () => { - const baseline = JSON.parse(read('security/npm-audit-baseline.json')); +test("security baseline file is present and ready for future exceptions", () => { + const baseline = JSON.parse(read("security/npm-audit-baseline.json")); assert.ok(Array.isArray(baseline.entries)); assert.equal(baseline.entries.length, 0); diff --git a/tsconfig.json b/tsconfig.json index 6acdf15..a1f1555 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,23 +1,23 @@ { - "compilerOptions": { - "module": "node16", - "types": ["node"], - "noImplicitReturns": true, - "noUnusedLocals": true, - "outDir": "build", - "sourceMap": true, - "strict": true, - "allowJs": true, - "target": "es2022", - "lib": ["es2022", "dom"], - "skipLibCheck": true, - "useDefineForClassFields": false, - "experimentalDecorators": true, - "emitDecoratorMetadata": true, - "esModuleInterop": true, - "noImplicitOverride": true, - }, - "compileOnSave": true, - "include": ["src"], - "exclude": ["src/cloud-functions/**/*", "src/utils/get-function-url.ts"] + "compilerOptions": { + "module": "node16", + "types": ["node"], + "noImplicitReturns": true, + "noUnusedLocals": true, + "outDir": "build", + "sourceMap": true, + "strict": true, + "allowJs": true, + "target": "es2022", + "lib": ["es2022", "dom"], + "skipLibCheck": true, + "useDefineForClassFields": false, + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "esModuleInterop": true, + "noImplicitOverride": true + }, + "compileOnSave": true, + "include": ["src"], + "exclude": ["src/cloud-functions/**/*", "src/utils/get-function-url.ts"] }