Files
xget/src/index.js
T
google-labs-jules[bot] 81c44f0c71 feat: add support for Hugging Face API operations
This change enables full support for Hugging Face API requests, including repository creation, by:
1. Identifying HF API requests via path patterns.
2. Allowing POST, PUT, PATCH, and DELETE methods for these requests.
3. Correctly forwarding request bodies and headers.
4. Skipping caching for API operations.

Fixes: huggingface_hub.errors.HfHubHTTPError: Client error '405 Method Not Allowed'
2026-01-06 14:53:29 +00:00

646 lines
29 KiB
JavaScript

/**
* Xget - High-performance acceleration engine for developer resources
* Copyright (C) 2025 Xi Xu
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*/
import { CONFIG, createConfig } from './config/index.js';
import { SORTED_PLATFORMS, transformPath } from './config/platforms.js';
import { configureAIHeaders, isAIInferenceRequest } from './protocols/ai.js';
import { configureHuggingFaceHeaders, isHuggingFaceAPIRequest } from './protocols/huggingface.js';
import {
fetchToken,
getScopeFromUrl,
handleDockerAuth,
parseAuthenticate,
responseUnauthorized
} from './protocols/docker.js';
import { configureGitHeaders, isGitLFSRequest, isGitRequest } from './protocols/git.js';
import { PerformanceMonitor, addPerformanceHeaders } from './utils/performance.js';
import { addSecurityHeaders, createErrorResponse } from './utils/security.js';
import { isDockerRequest, validateRequest } from './utils/validation.js';
/**
* Main request handler with comprehensive caching, retry logic, and security measures.
* @param {Request} request - The incoming HTTP request
* @param {object} env - Cloudflare Workers environment variables for runtime config overrides
* @param {ExecutionContext} ctx - Cloudflare Workers execution context for background tasks
* @returns {Promise<Response>} The HTTP response with appropriate headers and body
*/
async function handleRequest(request, env, ctx) {
let response;
const monitor = new PerformanceMonitor();
try {
// Create config with environment variable overrides
const config = env ? createConfig(env) : CONFIG;
const url = new URL(request.url);
const isDocker = isDockerRequest(request, url);
// Handle Docker API version check
if (isDocker && (url.pathname === '/v2/' || url.pathname === '/v2')) {
const headers = new Headers({
'Docker-Distribution-Api-Version': 'registry/2.0',
'Content-Type': 'application/json'
});
addSecurityHeaders(headers);
response = new Response('{}', { status: 200, headers });
}
// Redirect root path or invalid platforms to GitHub repository
else if (url.pathname === '/' || url.pathname === '') {
const HOME_PAGE_URL = 'https://github.com/xixu-me/Xget';
response = Response.redirect(HOME_PAGE_URL, 302);
} else {
const validation = validateRequest(request, url, config);
if (!validation.valid) {
response = createErrorResponse(
validation.error || 'Validation failed',
validation.status || 400
);
} else {
// Parse platform and path
let effectivePath = url.pathname;
// Handle container registry paths specially
if (isDocker) {
// For Docker requests, check if they have /cr/ prefix
// but allow /v2/auth which handles authentication
if (
!url.pathname.startsWith('/cr/') &&
!url.pathname.startsWith('/v2/cr/') &&
url.pathname !== '/v2/auth'
) {
response = createErrorResponse('container registry requests must use /cr/ prefix', 400);
} else {
// Remove /v2 from the path for container registry API consistency if present
effectivePath = url.pathname.replace(/^\/v2/, '');
}
}
if (!response) {
// Handle Docker authentication explicitly
if (isDocker && url.pathname === '/v2/auth') {
response = await handleDockerAuth(request, url, config);
} else {
// Platform detection using transform patterns
// Use pre-computed sorted platforms
const platform =
SORTED_PLATFORMS.find(key => {
const expectedPrefix = `/${key.replace('-', '/')}/`;
return effectivePath.startsWith(expectedPrefix);
}) || effectivePath.split('/')[1];
if (!platform || !config.PLATFORMS[platform]) {
const HOME_PAGE_URL = 'https://github.com/xixu-me/Xget';
response = Response.redirect(HOME_PAGE_URL, 302);
} else {
// Check if the path only contains the platform prefix without any actual resource path
const platformPath = `/${platform.replace(/-/g, '/')}`;
if (effectivePath === platformPath || effectivePath === `${platformPath}/`) {
const HOME_PAGE_URL = 'https://github.com/xixu-me/Xget';
response = Response.redirect(HOME_PAGE_URL, 302);
} else {
// Transform URL based on platform using unified logic
const targetPath = transformPath(effectivePath, platform);
// For container registries, ensure we add the /v2 prefix for the Docker API
let finalTargetPath;
if (platform.startsWith('cr-')) {
finalTargetPath = `/v2${targetPath}`;
} else {
finalTargetPath = targetPath;
}
const targetUrl = `${config.PLATFORMS[platform]}${finalTargetPath}${url.search}`;
const authorization = request.headers.get('Authorization');
// Check if this is a Git operation
const isGit = isGitRequest(request, url);
// Check if this is a Git LFS operation
const isGitLFS = isGitLFSRequest(request, url);
// Check if this is an AI inference request
const isAI = isAIInferenceRequest(request, url);
// Check if this is a Hugging Face API request
const isHF = isHuggingFaceAPIRequest(request, url);
// Check cache first (skip cache for Git, Git LFS, Docker, AI inference, and HF API operations)
/** @type {Cache | null} */
// @ts-ignore - Cloudflare Workers cache API
const cache =
typeof caches !== 'undefined' && /** @type {any} */ (caches).default // eslint-disable-line jsdoc/reject-any-type
? /** @type {any} */ (caches).default // eslint-disable-line jsdoc/reject-any-type
: null;
if (cache && !isGit && !isGitLFS && !isDocker && !isAI && !isHF) {
try {
// For Range requests, try cache match first
const cacheKey = new Request(targetUrl, {
method: 'GET',
headers: request.headers
});
const cachedResponse = await cache.match(cacheKey);
if (cachedResponse) {
monitor.mark('cache_hit');
response = cachedResponse;
} else {
// If Range request missed cache, try with original request to see if we have full content cached
const rangeHeader = request.headers.get('Range');
if (rangeHeader) {
const fullContentKey = new Request(targetUrl, {
method: 'GET', // Always use GET method for cache key consistency
headers: new Headers(
[...request.headers.entries()].filter(
([k]) => k.toLowerCase() !== 'range'
)
)
});
const fullCachedResponse = await cache.match(fullContentKey);
if (fullCachedResponse) {
monitor.mark('cache_hit_full_content');
response = fullCachedResponse;
}
}
}
} catch (cacheError) {
console.warn('Cache API unavailable:', cacheError);
}
}
if (!response) {
/** @type {RequestInit} */
const fetchOptions = {
method: request.method,
headers: new Headers(),
redirect: 'follow'
};
// Add body for POST/PUT/PATCH/DELETE requests (Git/Docker/AI/HF operations)
if (
['POST', 'PUT', 'PATCH', 'DELETE'].includes(request.method) &&
(isGit || isGitLFS || isDocker || isAI || isHF)
) {
fetchOptions.body = request.body;
}
// Cast headers to Headers for proper typing
const requestHeaders = /** @type {Headers} */ (fetchOptions.headers);
// Set appropriate headers for Git/Docker/AI/HF vs regular requests
if (isGit || isGitLFS || isDocker || isAI || isHF) {
// For Git/Docker/AI/HF operations, copy all headers from the original request
// This ensures protocol compliance
for (const [key, value] of request.headers.entries()) {
// Skip headers that might cause issues with proxying
if (
!['host', 'connection', 'upgrade', 'proxy-connection'].includes(
key.toLowerCase()
)
) {
requestHeaders.set(key, value);
}
}
// Configure protocol-specific headers using modular helpers
configureGitHeaders(requestHeaders, request, url, isGitLFS);
if (isAI) {
configureAIHeaders(requestHeaders, request);
}
if (isHF) {
configureHuggingFaceHeaders(requestHeaders, request);
}
} else {
// Regular file download headers
Object.assign(fetchOptions, {
cf: {
http3: true,
cacheTtl: config.CACHE_DURATION,
cacheEverything: true,
minify: {
javascript: true,
css: true,
html: true
},
preconnect: true
}
});
requestHeaders.set('Accept-Encoding', 'gzip, deflate, br');
requestHeaders.set('Connection', 'keep-alive');
requestHeaders.set('User-Agent', 'Wget/1.21.3');
requestHeaders.set('Origin', request.headers.get('Origin') || '*');
if (authorization) {
requestHeaders.set('Authorization', authorization);
}
const rangeHeader = request.headers.get('Range');
const isMediaFile = targetUrl.match(
/\.(mp4|avi|mkv|mov|wmv|flv|webm|mp3|wav|flac|aac|ogg|jpg|jpeg|png|gif|bmp|svg|pdf|zip|rar|7z|tar|gz|bz2|xz)$/i
);
if (isMediaFile || rangeHeader) {
requestHeaders.set('Accept-Encoding', 'identity');
}
if (rangeHeader) {
requestHeaders.set('Range', rangeHeader);
}
}
// Implement retry mechanism
let attempts = 0;
while (attempts < config.MAX_RETRIES) {
try {
monitor.mark(`attempt_${attempts}`);
const controller = new AbortController();
const timeoutId = setTimeout(
() => controller.abort(),
config.TIMEOUT_SECONDS * 1000
);
const finalFetchOptions = {
...fetchOptions,
signal: controller.signal
};
// Special handling for Docker redirects to avoid leaking Auth headers to S3 (blobs)
if (isDocker) {
finalFetchOptions.redirect = 'manual';
}
// Special handling for HEAD requests to ensure Content-Length header
if (request.method === 'HEAD') {
response = await fetch(targetUrl, finalFetchOptions);
if (response.ok && !response.headers.get('Content-Length')) {
const rangeHeaders = new Headers(requestHeaders);
rangeHeaders.set('Range', 'bytes=0-0');
const rangeResponse = await fetch(targetUrl, {
...finalFetchOptions,
method: 'GET',
headers: rangeHeaders
});
let contentLength = null;
if (rangeResponse.status === 206) {
const contentRange = rangeResponse.headers.get('Content-Range');
if (contentRange) {
const match = contentRange.match(/bytes\s+\d+-\d+\/(\d+)/);
if (match) {
[, contentLength] = match;
}
}
} else if (rangeResponse.ok) {
contentLength = rangeResponse.headers.get('Content-Length');
if (!contentLength) {
const sizeLimit = 50 * 1024 * 1024;
const contentLengthHint = rangeResponse.headers.get('Content-Length');
if (
!contentLengthHint ||
parseInt(contentLengthHint, 10) < sizeLimit
) {
try {
const arrayBuffer = await rangeResponse.arrayBuffer();
contentLength = arrayBuffer.byteLength.toString();
} catch (error) {
console.warn(
'Could not buffer response to get Content-Length:',
error
);
}
}
}
}
if (contentLength) {
const headHeaders = new Headers(response.headers);
headHeaders.set('Content-Length', contentLength);
response = new Response(null, {
status: response.status,
statusText: response.statusText,
headers: headHeaders
});
}
}
} else {
response = await fetch(targetUrl, finalFetchOptions);
}
clearTimeout(timeoutId);
// Handle manual redirect for Docker
if (isDocker && (response.status === 301 || response.status === 302 || response.status === 307)) {
const location = response.headers.get('Location');
if (location) {
// Fetch the new location without Authorization header
// Cloudflare Workers fetch should follow this automatically if we used 'follow',
// but we used 'manual' to strip headers.
const redirectHeaders = new Headers(finalFetchOptions.headers);
redirectHeaders.delete('Authorization');
response = await fetch(location, {
...finalFetchOptions,
headers: redirectHeaders,
redirect: 'follow' // Follow subsequent redirects normally
});
}
}
if (response.ok || response.status === 206) {
monitor.mark('success');
break;
}
// For container registry, handle authentication challenges more intelligently
if (isDocker && response.status === 401) {
monitor.mark('docker_auth_challenge');
const authenticateStr = response.headers.get('WWW-Authenticate');
// Calculate scope for upstream token fetch
let scope = getScopeFromUrl(url, effectivePath, platform);
if (authenticateStr) {
try {
const wwwAuthenticate = parseAuthenticate(authenticateStr);
// Try to get a token for public access (without authorization)
const tokenResponse = await fetchToken(
wwwAuthenticate,
scope || '',
''
);
if (tokenResponse.ok) {
const tokenData = await tokenResponse.json();
if (tokenData.token) {
const retryHeaders = new Headers(requestHeaders);
retryHeaders.set('Authorization', `Bearer ${tokenData.token}`);
const retryOptions = {
...finalFetchOptions,
headers: retryHeaders
};
// Also use manual redirect for retry
if (isDocker) {
retryOptions.redirect = 'manual';
}
let retryResponse = await fetch(targetUrl, retryOptions);
// Handle manual redirect for retry
if (isDocker && (retryResponse.status === 301 || retryResponse.status === 302 || retryResponse.status === 307)) {
const location = retryResponse.headers.get('Location');
if (location) {
const redirectHeaders = new Headers(retryOptions.headers);
redirectHeaders.delete('Authorization');
retryResponse = await fetch(location, {
...retryOptions,
headers: redirectHeaders,
redirect: 'follow'
});
}
}
if (retryResponse.ok) {
response = retryResponse;
monitor.mark('success');
break;
}
}
}
} catch (error) {
console.warn('Token fetch failed:', error);
}
}
response = responseUnauthorized(url);
break;
}
if (response.status >= 400 && response.status < 500) {
monitor.mark('client_error');
break;
}
attempts++;
if (attempts < config.MAX_RETRIES) {
await new Promise(resolve =>
setTimeout(resolve, config.RETRY_DELAY_MS * attempts)
);
}
} catch (error) {
attempts++;
if (error instanceof Error && error.name === 'AbortError') {
response = createErrorResponse('Request timeout', 408);
break;
}
if (attempts >= config.MAX_RETRIES) {
const message = error instanceof Error ? error.message : String(error);
response = createErrorResponse(
`Failed after ${config.MAX_RETRIES} attempts: ${message}`,
500,
true
);
break;
}
await new Promise(resolve =>
setTimeout(resolve, config.RETRY_DELAY_MS * attempts)
);
}
}
if (!response) {
response = createErrorResponse(
'No response received after all retry attempts',
500,
true
);
} else if (!response.ok && response.status !== 206) {
if (isDocker && response.status === 401) {
// Handle Docker 401 responses that might not have been caught by the retry loop
const isCustomError =
response.headers.get('content-type') === 'application/json' &&
(await response.clone().text()).includes('UNAUTHORIZED');
if (!isCustomError) {
const errorText = await response.text().catch(() => '');
response = createErrorResponse(
`Authentication required for this container registry resource. This may be a private repository. Original error: ${errorText}`,
401,
true
);
}
} else {
const errorText = await response.text().catch(() => 'Unknown error');
response = createErrorResponse(
`Upstream server error (${response.status}): ${errorText}`,
response.status,
true
);
}
} else {
// Success case processing (rewriting URLs etc)
let responseBody = response.body;
if (
platform === 'pypi' &&
response.headers.get('content-type')?.includes('text/html')
) {
const originalText = await response.text();
const rewrittenText = originalText.replace(
/https:\/\/files\.pythonhosted\.org/g,
`${url.origin}/pypi/files`
);
responseBody = new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(rewrittenText));
controller.close();
}
});
}
if (
platform === 'npm' &&
response.headers.get('content-type')?.includes('application/json')
) {
const originalText = await response.text();
const rewrittenText = originalText.replace(
/https:\/\/registry.npmjs.org\/([^/]+)/g,
`${url.origin}/npm/$1`
);
responseBody = new ReadableStream({
start(controller) {
controller.enqueue(new TextEncoder().encode(rewrittenText));
controller.close();
}
});
}
const headers = new Headers(response.headers);
if (!isGit && !isDocker) {
headers.set('Cache-Control', `public, max-age=${config.CACHE_DURATION}`);
headers.set('X-Content-Type-Options', 'nosniff');
headers.set('Accept-Ranges', 'bytes');
if (!headers.has('Content-Length') && response.status === 200) {
try {
const contentLength = response.headers.get('Content-Length');
if (contentLength) {
headers.set('Content-Length', contentLength);
}
} catch (error) {
console.warn('Could not set Content-Length header:', error);
}
}
addSecurityHeaders(headers);
}
response = new Response(responseBody, {
status: response.status,
headers
});
// Cache success logic
if (
cache &&
!isGit &&
!isGitLFS &&
!isDocker &&
!isAI &&
request.method === 'GET' &&
response.ok &&
response.status === 200
) {
const rangeHeader = request.headers.get('Range');
const cacheKey = rangeHeader
? new Request(targetUrl, {
method: 'GET',
headers: new Headers(
[...request.headers.entries()].filter(
([k]) => k.toLowerCase() !== 'range'
)
)
})
: new Request(targetUrl, { method: 'GET' });
try {
if (ctx && typeof ctx.waitUntil === 'function') {
ctx.waitUntil(cache.put(cacheKey, response.clone()));
} else {
cache.put(cacheKey, response.clone()).catch(error => {
console.warn('Cache put failed:', error);
});
}
if (rangeHeader && response.status === 200) {
const rangedResponse = await cache.match(
new Request(targetUrl, {
method: 'GET',
headers: request.headers
})
);
if (rangedResponse) {
monitor.mark('range_cache_hit_after_full_cache');
response = rangedResponse;
}
}
} catch (cacheError) {
console.warn('Cache put/match failed:', cacheError);
}
}
}
}
}
}
}
}
}
}
} catch (error) {
console.error('Error handling request:', error);
const message = error instanceof Error ? error.message : String(error);
response = createErrorResponse(`Internal Server Error: ${message}`, 500, true);
}
// Ensure performance headers are added to the final response
monitor.mark('complete');
const isGit = isGitRequest(request, new URL(request.url));
const isDocker = isDockerRequest(request, new URL(request.url));
const isAI = isAIInferenceRequest(request, new URL(request.url));
const isGitLFS = isGitLFSRequest(request, new URL(request.url));
const isHF = isHuggingFaceAPIRequest(request, new URL(request.url));
return isGit || isGitLFS || isDocker || isAI || isHF
? response
: addPerformanceHeaders(response, monitor);
}
export default {
/**
* Main Worker entry point.
* @param {Request} request
* @param {object} env
* @param {ExecutionContext} ctx
*/
fetch(request, env, ctx) {
return handleRequest(request, env, ctx);
}
};