From 17f94416d673cee3fc7bb68f5215a172da63a08d Mon Sep 17 00:00:00 2001 From: Xi Xu Date: Sun, 9 Nov 2025 15:51:39 +0800 Subject: [PATCH] Add Git LFS protocol support and tests Introduces detection and handling for Git LFS (Large File Storage) operations, including LFS-specific endpoints, headers, and user agents. Updates request validation, header management, and cache logic to properly support LFS requests and ensure real-time data synchronization. Adds comprehensive integration and unit tests for LFS scenarios, and documents the new behavior in CLAUDE.md. --- CLAUDE.md | 8 +++ src/index.js | 78 ++++++++++++++++++++++++++-- test/integration.test.js | 92 +++++++++++++++++++++++++++++++++ test/utils.test.js | 108 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 281 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5d778ca..617d0f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -89,6 +89,14 @@ Xget is a high-performance, secure acceleration engine for developer resources, - Allows POST method, sets Git-specific headers - Skips caching to ensure real-time data +**Git LFS (Large File Storage) Operations** ([src/index.js](src/index.js):104-141) + +- Detected via LFS-specific endpoints (`/info/lfs`, `/objects/batch`, `/objects/{oid}`) +- Detected via LFS headers (`Accept: application/vnd.git-lfs+json`) or User-Agent (`git-lfs/`) +- Supports batch API for efficient object transfers +- Sets appropriate content-type headers for LFS operations +- Skips caching to ensure real-time data synchronization + **Docker/Container Registries** ([src/index.js](src/index.js):42-65) - Detected via `/v2/` paths, User-Agent, or Accept headers diff --git a/src/index.js b/src/index.js index 064931d..3d06485 100644 --- a/src/index.js +++ b/src/index.js @@ -101,6 +101,45 @@ function isGitRequest(request, url) { return false; } +/** + * Check if the request is a Git LFS operation + * @param {Request} request - The incoming request object + * @param {URL} url - Parsed URL object + * @returns {boolean} True if this is a Git LFS operation + */ +function isGitLFSRequest(request, url) { + // Check for LFS-specific endpoints + if (url.pathname.includes('/info/lfs')) { + return true; + } + + if (url.pathname.includes('/objects/batch')) { + return true; + } + + // Check for LFS object storage endpoints (SHA-256 hash is 64 hex characters) + if (url.pathname.match(/\/objects\/[a-fA-F0-9]{64}$/)) { + return true; + } + + // Check for LFS-specific headers + const accept = request.headers.get('Accept') || ''; + const contentType = request.headers.get('Content-Type') || ''; + + if (accept.includes('application/vnd.git-lfs') || + contentType.includes('application/vnd.git-lfs')) { + return true; + } + + // Check for LFS user agent + const userAgent = request.headers.get('User-Agent') || ''; + if (userAgent.includes('git-lfs')) { + return true; + } + + return false; +} + /** * Check if the request is for an AI inference provider * @param {Request} request - The incoming request object @@ -153,13 +192,14 @@ function isAIInferenceRequest(request, url) { * @returns {{valid: boolean, error?: string, status?: number}} Validation result */ function validateRequest(request, url, config = CONFIG) { - // Allow POST method for Git, Docker, and AI inference operations + // Allow POST method for Git, Git LFS, Docker, and AI inference operations const isGit = isGitRequest(request, url); + const isGitLFS = isGitLFSRequest(request, url); const isDocker = isDockerRequest(request, url); const isAI = isAIInferenceRequest(request, url); const allowedMethods = - isGit || isDocker || isAI + isGit || isGitLFS || isDocker || isAI ? ['GET', 'HEAD', 'POST', 'PUT', 'PATCH'] : config.SECURITY.ALLOWED_METHODS; @@ -373,16 +413,19 @@ async function handleRequest(request, env, ctx) { // Check if this is a Git operation const isGit = isGitRequest(request, url); + // Check if this is a Git LFS operation + const isGitLFS = isGitLFSRequest(request, url); + // Check if this is an AI inference request const isAI = isAIInferenceRequest(request, url); - // Check cache first (skip cache for Git, Docker, and AI inference operations) + // Check cache first (skip cache for Git, Git LFS, Docker, and AI inference operations) /** @type {Cache} */ // @ts-ignore - Cloudflare Workers cache API const cache = caches.default; let response; - if (!isGit && !isDocker && !isAI) { + if (!isGit && !isGitLFS && !isDocker && !isAI) { // For Range requests, try cache match first const cacheKey = new Request(targetUrl, request); response = await cache.match(cacheKey); @@ -453,6 +496,30 @@ async function handleRequest(request, env, ctx) { } } + // Set Git LFS-specific headers + if (isGitLFS) { + if (!requestHeaders.has('User-Agent')) { + requestHeaders.set('User-Agent', 'git-lfs/3.0.0 (GitHub; darwin amd64; go 1.17.2)'); + } + + // For LFS batch API requests + if (url.pathname.includes('/objects/batch')) { + if (!requestHeaders.has('Accept')) { + requestHeaders.set('Accept', 'application/vnd.git-lfs+json'); + } + if (request.method === 'POST' && !requestHeaders.has('Content-Type')) { + requestHeaders.set('Content-Type', 'application/vnd.git-lfs+json'); + } + } + + // For LFS object transfers + if (url.pathname.match(/\/objects\/[a-fA-F0-9]{64}$/)) { + if (!requestHeaders.has('Accept')) { + requestHeaders.set('Accept', 'application/octet-stream'); + } + } + } + // For AI inference requests, ensure proper content type and headers if (isAI) { // Ensure JSON content type for AI API requests if not already set @@ -766,11 +833,12 @@ async function handleRequest(request, env, ctx) { headers }); - // Cache successful responses (skip caching for Git, Docker, and AI inference operations) + // Cache successful responses (skip caching for Git, Git LFS, Docker, and AI inference operations) // Only cache GET and HEAD requests to avoid "Cannot cache response to non-GET request" errors // IMPORTANT: Only cache 200 responses, NOT 206 responses (Cloudflare Workers Cache API rejects 206) if ( !isGit && + !isGitLFS && !isDocker && !isAI && ['GET', 'HEAD'].includes(request.method) && diff --git a/test/integration.test.js b/test/integration.test.js index 238d790..202d886 100644 --- a/test/integration.test.js +++ b/test/integration.test.js @@ -107,6 +107,98 @@ describe('Integration Tests', () => { }); }); + describe('Git LFS Protocol Integration', () => { + it('should handle LFS info/lfs requests', async () => { + const testUrl = 'https://example.com/gh/microsoft/vscode.git/info/lfs'; + const response = await SELF.fetch(testUrl, { + headers: { + 'User-Agent': 'git-lfs/3.0.0 (GitHub; darwin amd64; go 1.17.2)' + } + }); + + expect([200, 301, 302, 404]).toContain(response.status); + }); + + it('should handle LFS batch API requests', async () => { + const testUrl = 'https://example.com/gh/microsoft/vscode.git/objects/batch'; + const response = await SELF.fetch(testUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.git-lfs+json', + 'Accept': 'application/vnd.git-lfs+json', + 'User-Agent': 'git-lfs/3.0.0' + }, + body: JSON.stringify({ + operation: 'download', + objects: [ + { + oid: 'a1b2c3d4e5f6789012345678901234567890123456789012345678901234abcd', + size: 1024 + } + ] + }) + }); + + expect([200, 301, 302, 400, 403, 404]).toContain(response.status); + }); + + it('should handle LFS object download requests', async () => { + const testUrl = 'https://example.com/gh/microsoft/vscode.git/objects/a1b2c3d4e5f6789012345678901234567890123456789012345678901234abcd'; + const response = await SELF.fetch(testUrl, { + headers: { + 'User-Agent': 'git-lfs/3.0.0', + 'Accept': 'application/octet-stream' + } + }); + + expect([200, 301, 302, 403, 404]).toContain(response.status); + }); + + it('should preserve LFS-specific headers', async () => { + const testUrl = 'https://example.com/gh/test/repo.git/objects/batch'; + const response = await SELF.fetch(testUrl, { + method: 'POST', + headers: { + 'User-Agent': 'git-lfs/3.0.0', + 'Accept': 'application/vnd.git-lfs+json', + 'Content-Type': 'application/vnd.git-lfs+json' + }, + body: '{}' + }); + + // Should not reject LFS-specific headers + expect(response.status).not.toBe(400); + }); + + it('should skip caching for LFS requests', async () => { + const testUrl = 'https://example.com/gh/test/repo.git/info/lfs'; + + // First request + const response1 = await SELF.fetch(testUrl, { + headers: { + 'User-Agent': 'git-lfs/3.0.0' + } + }); + + // Second request - should not be cached + const response2 = await SELF.fetch(testUrl, { + headers: { + 'User-Agent': 'git-lfs/3.0.0' + } + }); + + // Both requests should go to origin (no cache hit) + const metrics1 = response1.headers.get('X-Performance-Metrics'); + const metrics2 = response2.headers.get('X-Performance-Metrics'); + + // Verify that neither indicates a cache hit + if (metrics1 && metrics2) { + expect(metrics1).not.toContain('cache_hit'); + expect(metrics2).not.toContain('cache_hit'); + } + }); + }); + describe('Caching Integration', () => { it('should cache responses appropriately', async () => { const testUrl = 'https://example.com/gh/test/repo/static-file.txt'; diff --git a/test/utils.test.js b/test/utils.test.js index 1b3a181..5c96338 100644 --- a/test/utils.test.js +++ b/test/utils.test.js @@ -34,6 +34,39 @@ function isGitRequest(request, url) { return false; } +function isGitLFSRequest(request, url) { + // Check for LFS-specific endpoints + if (url.pathname.includes('/info/lfs')) { + return true; + } + + if (url.pathname.includes('/objects/batch')) { + return true; + } + + // Check for LFS object storage endpoints (SHA-256 hash is 64 hex characters) + if (url.pathname.match(/\/objects\/[a-fA-F0-9]{64}$/)) { + return true; + } + + // Check for LFS-specific headers + const accept = request.headers.get('Accept') || ''; + const contentType = request.headers.get('Content-Type') || ''; + + if (accept.includes('application/vnd.git-lfs') || + contentType.includes('application/vnd.git-lfs')) { + return true; + } + + // Check for LFS user agent + const userAgent = request.headers.get('User-Agent') || ''; + if (userAgent.includes('git-lfs')) { + return true; + } + + return false; +} + function validateRequest(request, url) { const CONFIG = { SECURITY: { @@ -132,6 +165,81 @@ describe('Utility Functions', () => { }); }); + describe('isGitLFSRequest', () => { + it('should identify LFS info/lfs requests', () => { + const request = new Request('https://example.com/repo.git/info/lfs'); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(true); + }); + + it('should identify LFS batch API requests', () => { + const request = new Request('https://example.com/repo.git/objects/batch', { + method: 'POST', + headers: { 'Content-Type': 'application/vnd.git-lfs+json' } + }); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(true); + }); + + it('should identify LFS object storage requests by path', () => { + const request = new Request('https://example.com/repo.git/objects/a1b2c3d4e5f6789012345678901234567890123456789012345678901234abcd'); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(true); + }); + + it('should identify LFS requests by Accept header', () => { + const request = new Request('https://example.com/repo.git/objects/batch', { + headers: { 'Accept': 'application/vnd.git-lfs+json' } + }); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(true); + }); + + it('should identify LFS requests by Content-Type header', () => { + const request = new Request('https://example.com/repo.git/objects/batch', { + method: 'POST', + headers: { 'Content-Type': 'application/vnd.git-lfs+json' } + }); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(true); + }); + + it('should identify LFS requests by User-Agent', () => { + const request = new Request('https://example.com/repo.git', { + headers: { 'User-Agent': 'git-lfs/3.0.0 (GitHub; darwin amd64; go 1.17.2)' } + }); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(true); + }); + + it('should not identify regular file requests as LFS', () => { + const request = new Request('https://example.com/repo/file.txt'); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(false); + }); + + it('should not identify standard Git requests as LFS', () => { + const request = new Request('https://example.com/repo.git/info/refs'); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(false); + }); + + it('should handle edge cases gracefully', () => { + const request = new Request('https://example.com/'); + const url = new URL(request.url); + + expect(isGitLFSRequest(request, url)).toBe(false); + }); + }); + describe('validateRequest', () => { it('should allow GET requests', () => { const request = new Request('https://example.com/test', { method: 'GET' });