diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..d6c7bff --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,207 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +DeepLX is a serverless translation service optimized for Cloudflare Workers. It provides a free alternative to translation APIs by proxying requests to DeepL and Google Translate services through intelligent load balancing, rate limiting, and caching mechanisms. + +## Development Commands + +### Running & Deployment +- `npm run dev` - Start local development server with Wrangler +- `npm run deploy` - Deploy to Cloudflare Workers production +- `npm run cf-typegen` - Generate TypeScript types for Cloudflare Workers bindings + +### Testing +- `npm test` - Run all tests +- `npm run test:unit` - Run unit tests only (tests in tests/lib/) +- `npm run test:integration` - Run integration tests (tests/integration/) +- `npm run test:performance` - Run performance tests (tests/performance/) +- `npm run test:coverage` - Generate coverage report +- `npm run test:watch` - Run tests in watch mode +- `npm run test:debug` - Run tests with Node debugging enabled + +### Code Quality +- `npm run lint` - Type-check with TypeScript (no emit, runs `tsc --noEmit`) + +## Architecture + +### Request Flow +1. Request arrives at Hono router (src/index.ts) +2. CORS preflight handling (if OPTIONS) +3. Security middleware validates input and extracts client IP +4. Rate limiting checks (two-level: client IP + proxy backend) +5. Cache lookup (two-level: in-memory + KV) +6. If cache miss: + - Proxy manager selects available proxy endpoint + - Circuit breaker checks if proxy is healthy + - Retry logic with exponential backoff + - Translation query to provider (DeepL or Google) +7. Cache successful translation +8. Return standardized response + +### Core Components + +#### Translation Providers +- **DeepL Query Engine** ([src/lib/query.ts](src/lib/query.ts)): Main translation logic for DeepL using JSONRPC API +- **Google Translate Service** ([src/lib/services/googleTranslate.ts](src/lib/services/googleTranslate.ts)): Google Translate integration + +#### Support Systems +- **Two-Level Cache** ([src/lib/cache.ts](src/lib/cache.ts)): In-memory Map + Cloudflare KV with 1-hour TTL +- **Token Bucket Rate Limiter** ([src/lib/rateLimit.ts](src/lib/rateLimit.ts)): Dual rate limiting (per-client IP + per-proxy) with dynamic limits based on proxy count +- **Circuit Breaker** ([src/lib/circuitBreaker.ts](src/lib/circuitBreaker.ts)): Prevents cascade failures by temporarily blocking failing proxy endpoints (states: CLOSED/OPEN/HALF_OPEN) +- **Proxy Manager** ([src/lib/proxyManager.ts](src/lib/proxyManager.ts)): Random proxy selection with browser fingerprinting (User-Agent, Accept-Language rotation) +- **Retry Logic** ([src/lib/retryLogic.ts](src/lib/retryLogic.ts)): Exponential backoff retry mechanism (max 3 retries, 1s initial delay, 2x backoff factor) + +#### Configuration +All configurable constants are centralized in [src/lib/config.ts](src/lib/config.ts): +- Rate limits: `RATE_LIMIT_CONFIG` (dynamic based on proxy count) +- Cache TTL: `CACHE_CONFIG` +- Retry settings: `DEFAULT_RETRY_CONFIG` +- Payload limits: `PAYLOAD_LIMITS` +- Request timeout: `REQUEST_TIMEOUT` + +### API Endpoints + +The application exposes three POST endpoints (defined in [src/index.ts](src/index.ts)): +- `/deepl` - DeepL translation (recommended) +- `/google` - Google Translate +- `/translate` - Legacy endpoint (uses DeepL for backward compatibility) +- `/debug` - Debug endpoint (only available when `DEBUG_MODE=true`) + +All endpoints use the same `handleTranslation()` function with a provider parameter. + +### Environment Configuration + +Required environment variables in [wrangler.jsonc](wrangler.jsonc): +- `DEBUG_MODE` - Enable debug endpoint (default: "false") +- `PROXY_URLS` - Comma-separated list of XDPL proxy endpoints for DeepL + +Required KV namespaces: +- `CACHE_KV` - Translation result cache +- `RATE_LIMIT_KV` - Rate limit token buckets + +Required bindings: +- `ANALYTICS` - Cloudflare Analytics Engine dataset + +### Scheduled Tasks + +The worker includes a scheduled event handler that runs every 5 minutes (configured in `wrangler.jsonc` triggers): +- Clears in-memory cache to prevent memory leaks +- Called via `handleScheduled()` in [src/index.ts](src/index.ts) + +## Key Implementation Details + +### Rate Limiting Strategy +- **Client-level**: Token bucket with dynamic limit = `(proxy_count × 8 requests/sec) × 60 = tokens/minute` +- **Proxy-level**: 8 tokens/sec per proxy with 16 token burst capacity +- Uses two-level caching (in-memory 5s TTL + KV 1h TTL) for performance +- See `checkCombinedRateLimit()` in [src/lib/rateLimit.ts](src/lib/rateLimit.ts) + +### Caching Strategy +- **In-memory cache**: JavaScript Map for fast lookups (cleared every 5 minutes by scheduled task) +- **KV cache**: Cloudflare KV for persistence (1-hour TTL) +- Cache key generation: hash of `text:source_lang:target_lang:provider` +- See [src/lib/cache.ts](src/lib/cache.ts) + +### Circuit Breaker Pattern +- Opens after 5 consecutive failures +- 30-second recovery timeout +- Requires 3 consecutive successes to close +- One circuit breaker instance per proxy URL +- See [src/lib/circuitBreaker.ts](src/lib/circuitBreaker.ts) + +### Security Features +- Input validation and sanitization in [src/lib/security.ts](src/lib/security.ts) and [src/lib/validation.ts](src/lib/validation.ts) +- Maximum text length: 5000 characters (configurable in `PAYLOAD_LIMITS`) +- Language code validation with whitelist +- Client IP extraction from CF-Connecting-IP or X-Forwarded-For headers +- CORS handling via `handleCORSPreflight()` + +## Testing Strategy + +Tests are organized in the [tests/](tests/) directory: +- `tests/lib/` - Unit tests for individual library modules +- `tests/integration/` - Integration tests for API endpoints +- `tests/performance/` - Performance and load tests +- `tests/setup.ts` - Jest setup file +- `tests/utils/testHelpers.ts` - Shared test utilities + +Test configuration in [jest.config.js](jest.config.js): +- Uses `ts-jest` preset for TypeScript +- 30-second test timeout +- Coverage collected from `src/**/*.ts` + +When writing tests: +- Mock Cloudflare Workers bindings (KV, Analytics) +- Use `testHelpers.ts` utilities for creating mock environments +- Integration tests should test full request/response cycle +- Performance tests should validate response time constraints + +## TypeScript Configuration + +- **Target**: ES2020 with WebWorker lib +- **Module**: ESNext with bundler resolution (Cloudflare Workers uses bundling) +- **Strict mode**: Enabled +- Types: `@cloudflare/workers-types`, `jest` +- Entry point: [src/index.ts](src/index.ts) + +## Contributing Guidelines + +From [CONTRIBUTING.md](CONTRIBUTING.md): + +### Branch naming: +- `feature/description` - New features +- `fix/description` - Bug fixes +- `docs/description` - Documentation +- `refactor/description` - Code refactoring +- `test/description` - Test improvements + +### Commit format: +Use conventional commits: `type(scope): description` +- Types: feat, fix, docs, style, refactor, test, chore + +### Code style: +- 2 spaces indentation +- Single quotes for strings +- Trailing commas in multiline structures +- Lines under 100 characters +- Explicit types for function parameters/returns +- Avoid `any` type + +### Before submitting PR: +1. Run `npm test` (all tests must pass) +2. Run `npm run lint` (type check) +3. Add tests for new functionality +4. Update documentation if needed +5. Ensure code coverage doesn't decrease + +## Performance Optimization + +Optimized for Cloudflare Workers serverless environment: +- Minimize memory allocations (cache cleanup every 5 minutes) +- Efficient token bucket algorithm (O(1) rate limit checks) +- Two-level caching reduces KV read latency +- Browser fingerprinting uses pre-defined arrays (no runtime generation) +- Async KV writes don't block responses (fire-and-forget pattern in rate limiter) + +## Common Debugging Scenarios + +### Enable Debug Mode +Set `DEBUG_MODE=true` in wrangler.jsonc to enable the `/debug` endpoint for request inspection. + +### Rate Limit Issues +- Check dynamic rate limits with `getDynamicRateLimits(env)` in [src/lib/rateLimit.ts](src/lib/rateLimit.ts) +- Rate limits scale with proxy count: more proxies = higher client limits +- Clear rate limit cache: delete keys matching `rate_limit:*` in `RATE_LIMIT_KV` + +### Circuit Breaker Tripped +- Circuit opens after 5 consecutive proxy failures +- Wait 30 seconds for automatic HALF_OPEN state +- Check proxy health in proxy manager logs + +### Cache Issues +- In-memory cache is cleared every 5 minutes by scheduled task +- KV cache has 1-hour TTL +- Cache key collision: verify `generateCacheKey()` produces unique keys for different inputs diff --git a/package-lock.json b/package-lock.json index df2c40c..e9cd939 100644 --- a/package-lock.json +++ b/package-lock.json @@ -79,6 +79,7 @@ "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", @@ -626,7 +627,8 @@ "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20250722.0.tgz", "integrity": "sha512-pTY+A07DTSacgUBYcVEEb78/KG7THdcRpPqXLeH/A/LHHobAddgN4zyXBldsoZuzy7bD9tZYJW+wkcyR4k7fDA==", "dev": true, - "license": "MIT OR Apache-2.0" + "license": "MIT OR Apache-2.0", + "peer": true }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", @@ -1546,6 +1548,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -2554,6 +2557,7 @@ "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -4154,6 +4158,7 @@ "integrity": "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -4192,6 +4197,7 @@ "integrity": "sha512-B06u0wXkEd+o5gOCMl/ZHl5cfpYbDZKAT+HWTL+Hws6jWu7dCiqBBXXXzMFcFVJb8D4ytAnYmxJA83uwOQRSsg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "defu": "^6.1.4", "exsolve": "^1.0.4", @@ -4290,6 +4296,7 @@ "dev": true, "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "bin": { "workerd": "bin/workerd" }, diff --git a/src/lib/proxyManager.ts b/src/lib/proxyManager.ts index 443c3dc..4c5ab4e 100644 --- a/src/lib/proxyManager.ts +++ b/src/lib/proxyManager.ts @@ -68,18 +68,23 @@ export async function selectProxy(env: Env): Promise { /** * Generate realistic browser fingerprint headers - * Creates randomized headers to mimic real browser requests + * Creates randomized headers to mimic real browser requests and prevent 503 errors + * Enhanced with comprehensive headers to bypass DeepL's bot detection * @returns Record - Object containing HTTP headers */ export function generateBrowserFingerprint(): Record { return { "User-Agent": USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)], + Accept: "*/*", "Accept-Language": ACCEPT_LANGUAGES[Math.floor(Math.random() * ACCEPT_LANGUAGES.length)], - Accept: "application/json, text/plain, */*", - "Accept-Encoding": "gzip, deflate, br", + "Accept-Encoding": "gzip, deflate, br, zstd", + Origin: "https://www.deepl.com", + Referer: "https://www.deepl.com/", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-site", DNT: "1", Connection: "keep-alive", - "Upgrade-Insecure-Requests": "1", }; } diff --git a/tests/utils/testHelpers.ts b/tests/utils/testHelpers.ts index 881dfbc..0ef5126 100644 --- a/tests/utils/testHelpers.ts +++ b/tests/utils/testHelpers.ts @@ -186,12 +186,6 @@ export function createTestEnvironment() { const env: Env = { CACHE_KV: cacheKV as any, RATE_LIMIT_KV: rateLimitKV as any, - - { status: 200 } - ) - ), - }), - } as any, ANALYTICS: { writeDataPoint: jest.fn(), } as any,