From ce22782652a4cb361bdf862b7ee3c6e05ca51f46 Mon Sep 17 00:00:00 2001 From: Xi Xu Date: Sun, 27 Jul 2025 15:07:02 +0800 Subject: [PATCH] Remove HTML content preservation utilities Deleted htmlUtils module and its related tests. Removed all references to HTML preservation functions from index.ts and query.ts, simplifying the query logic and eliminating HTML-specific handling. --- src/lib/htmlUtils.ts | 142 ------------------------------------ src/lib/index.ts | 1 - src/lib/query.ts | 30 +------- tests/lib/htmlUtils.test.ts | 128 -------------------------------- 4 files changed, 3 insertions(+), 298 deletions(-) delete mode 100644 src/lib/htmlUtils.ts delete mode 100644 tests/lib/htmlUtils.test.ts diff --git a/src/lib/htmlUtils.ts b/src/lib/htmlUtils.ts deleted file mode 100644 index 9e4a3b6..0000000 --- a/src/lib/htmlUtils.ts +++ /dev/null @@ -1,142 +0,0 @@ -/** - * HTML content preservation utilities for DeepLX API - * Handles HTML tags and entities to prevent corruption during translation - */ - -/** - * HTML tag and entity patterns for preservation - */ -const HTML_PATTERNS = { - // HTML tags (opening and closing) - TAGS: /<\/?[a-zA-Z][a-zA-Z0-9]*(?:\s+[^>]*)?>/g, - // HTML entities - ENTITIES: /&(?:[a-zA-Z][a-zA-Z0-9]*|#(?:\d+|x[0-9a-fA-F]+));/g, - // Common punctuation that might be converted - PUNCTUATION: /[:：]/g, -}; - -/** - * Placeholder patterns for temporary replacement - * Using a format that's less likely to be modified by translation services - */ -const PLACEHOLDER_PREFIX = "ĦĐŁXĦ"; -const PLACEHOLDER_SUFFIX = "ĦĐŁXĦ"; - -/** - * Storage for preserved content during translation - */ -interface PreservedContent { - placeholders: Map; - counter: number; -} - -/** - * Create a unique placeholder for preserved content - * @param content The content to preserve - * @param preserved The preservation storage object - * @returns The placeholder string - */ -function createPlaceholder( - content: string, - preserved: PreservedContent -): string { - // Use a more unique format that's less likely to be translated - const placeholder = `${PLACEHOLDER_PREFIX}${preserved.counter - .toString() - .padStart(3, "0")}${PLACEHOLDER_SUFFIX}`; - preserved.placeholders.set(placeholder, content); - preserved.counter++; - return placeholder; -} - -/** - * Preserve HTML tags and entities before translation - * Replaces HTML content with placeholders to prevent corruption - * @param text The text containing HTML content - * @returns Object with processed text and preservation data - */ -export function preserveHtmlContent(text: string): { - processedText: string; - preserved: PreservedContent; -} { - const preserved: PreservedContent = { - placeholders: new Map(), - counter: 0, - }; - - let processedText = text; - - // Preserve HTML tags - processedText = processedText.replace(HTML_PATTERNS.TAGS, (match) => { - return createPlaceholder(match, preserved); - }); - - // Preserve HTML entities - processedText = processedText.replace(HTML_PATTERNS.ENTITIES, (match) => { - return createPlaceholder(match, preserved); - }); - - // Preserve colons to prevent conversion to Chinese colons - processedText = processedText.replace(/:/g, (match) => { - return createPlaceholder(match, preserved); - }); - - return { processedText, preserved }; -} - -/** - * Restore preserved HTML content after translation - * Replaces placeholders with original HTML content - * @param translatedText The translated text with placeholders - * @param preserved The preservation data from preserveHtmlContent - * @returns The text with restored HTML content - */ -export function restoreHtmlContent( - translatedText: string, - preserved: PreservedContent -): string { - let restoredText = translatedText; - - // Restore all preserved content - handle case-insensitive matching - for (const [placeholder, originalContent] of preserved.placeholders) { - // Create case-insensitive regex to handle lowercase conversion - const regex = new RegExp( - placeholder.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), - "gi" - ); - restoredText = restoredText.replace(regex, originalContent); - } - - // Additional cleanup: fix any remaining Chinese colons that might have been introduced - restoredText = restoredText.replace(/：/g, ":"); - - return restoredText; -} - -/** - * Check if text contains HTML content that should be preserved - * @param text The text to check - * @returns True if the text contains HTML content - */ -export function containsHtmlContent(text: string): boolean { - return HTML_PATTERNS.TAGS.test(text) || HTML_PATTERNS.ENTITIES.test(text); -} - -/** - * Sanitize HTML content while preserving structure - * Removes potentially dangerous HTML while keeping formatting tags - * @param text The text to sanitize - * @returns Sanitized text with safe HTML preserved - */ -export function sanitizeHtmlContent(text: string): string { - // Allow only safe formatting tags - const SAFE_TAGS = - /^<\/?(?:strong|b|em|i|u|code|pre|span|div|p|br|hr)(?:\s+[^>]*)?>/i; - - return text.replace(HTML_PATTERNS.TAGS, (match) => { - if (SAFE_TAGS.test(match)) { - return match; // Keep safe tags - } - return ""; // Remove unsafe tags - }); -} diff --git a/src/lib/index.ts b/src/lib/index.ts index 31b991e..10f5010 100644 --- a/src/lib/index.ts +++ b/src/lib/index.ts @@ -9,7 +9,6 @@ import { query } from "./query"; export * from "./cache"; export * from "./circuitBreaker"; export * from "./errorHandler"; -export * from "./htmlUtils"; export * from "./proxyManager"; export * from "./rateLimit"; export * from "./retryLogic"; diff --git a/src/lib/query.ts b/src/lib/query.ts index eb72d6f..294b437 100644 --- a/src/lib/query.ts +++ b/src/lib/query.ts @@ -10,11 +10,6 @@ import { } from "./config"; import { API_URL, REQUEST_ALTERNATIVES } from "./const"; import { createErrorResponse } from "./errorHandler"; -import { - containsHtmlContent, - preserveHtmlContent, - restoreHtmlContent, -} from "./htmlUtils"; import { generateBrowserFingerprint, selectProxy } from "./proxyManager"; import { checkCombinedRateLimit } from "./rateLimit"; import { isRetryableError, RetryOptions, retryWithBackoff } from "./retryLogic"; @@ -279,7 +274,7 @@ function buildRequestBody(data: RequestParams) { */ async function query( params: RequestParams, - config?: Config & { env?: Env } + config?: Config & { env?: any } ): Promise { if (!params?.text) { return createStandardResponse( @@ -291,20 +286,6 @@ async function query( ); } - // Check if text contains HTML content and preserve it - const hasHtmlContent = containsHtmlContent(params.text); - let processedParams = params; - let htmlPreservation: any = null; - - if (hasHtmlContent) { - const preservation = preserveHtmlContent(params.text); - htmlPreservation = preservation.preserved; - processedParams = { - ...params, - text: preservation.processedText, - }; - } - const retryOptions: RetryOptions = { ...DEFAULT_RETRY_CONFIG, isRetryable: isRetryableError, @@ -339,7 +320,7 @@ async function query( const timeoutId = setTimeout(() => controller.abort(), REQUEST_TIMEOUT); try { - const requestBody = buildRequestBody(processedParams); + const requestBody = buildRequestBody(params); const response = await fetch(endpoint, { headers: { @@ -455,12 +436,7 @@ async function query( throw error; } - let translatedText = result.result.texts[0].text; - - // Restore HTML content if it was preserved - if (hasHtmlContent && htmlPreservation) { - translatedText = restoreHtmlContent(translatedText, htmlPreservation); - } + const translatedText = result.result.texts[0].text; return createStandardResponse( 200, diff --git a/tests/lib/htmlUtils.test.ts b/tests/lib/htmlUtils.test.ts deleted file mode 100644 index 3a8eb1d..0000000 --- a/tests/lib/htmlUtils.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Tests for HTML utilities - HTML content preservation functionality - */ - -import { - containsHtmlContent, - preserveHtmlContent, - restoreHtmlContent, - sanitizeHtmlContent, -} from "../../src/lib/htmlUtils"; - -describe("HTML Utils", () => { - describe("containsHtmlContent", () => { - it("should detect HTML tags", () => { - expect(containsHtmlContent("Hello")).toBe(true); - expect(containsHtmlContent("test")).toBe(true); - expect(containsHtmlContent("Hello world")).toBe(false); - }); - - it("should detect HTML entities", () => { - expect(containsHtmlContent("<test>")).toBe(true); - expect(containsHtmlContent("&")).toBe(true); - expect(containsHtmlContent("Hello world")).toBe(false); - }); - }); - - describe("preserveHtmlContent and restoreHtmlContent", () => { - it("should preserve and restore HTML tags", () => { - const originalText = "A: B"; - const { processedText, preserved } = preserveHtmlContent(originalText); - - // The processed text should not contain HTML tags - expect(processedText).not.toContain(""); - expect(processedText).not.toContain(""); - expect(processedText).not.toContain(""); - expect(processedText).not.toContain(""); - - // Should contain placeholders - expect(processedText).toContain("ĦĐŁXĦ"); - - // Restore should bring back original content - const restoredText = restoreHtmlContent(processedText, preserved); - expect(restoredText).toBe(originalText); - }); - - it("should preserve colons to prevent Chinese colon conversion", () => { - const originalText = "Hello: World"; - const { processedText, preserved } = preserveHtmlContent(originalText); - - // The processed text should not contain colons - expect(processedText).not.toContain(":"); - expect(processedText).toContain("ĦĐŁXĦ"); - - // Restore should bring back original colon - const restoredText = restoreHtmlContent(processedText, preserved); - expect(restoredText).toBe(originalText); - }); - - it("should handle complex HTML with multiple tags", () => { - const originalText = - "Bold and italic: code here"; - const { processedText, preserved } = preserveHtmlContent(originalText); - - // Should not contain any HTML tags - expect(processedText).not.toMatch(/<[^>]+>/); - expect(processedText).not.toContain(":"); - - // Should contain placeholders - expect(processedText).toContain("ĦĐŁXĦ"); - - // Restore should bring back all original content - const restoredText = restoreHtmlContent(processedText, preserved); - expect(restoredText).toBe(originalText); - }); - - it("should fix Chinese colons in translated text", () => { - const originalText = "Hello: World"; - const { processedText, preserved } = preserveHtmlContent(originalText); - - // Simulate translation that introduces Chinese colon - const translatedWithChineseColon = processedText.replace( - "World", - "世界：extra" - ); - - const restoredText = restoreHtmlContent( - translatedWithChineseColon, - preserved - ); - - // Should restore original colon and fix any Chinese colons - expect(restoredText).toContain("Hello: 世界:extra"); - expect(restoredText).not.toContain("："); - }); - - it("should handle placeholders being converted to lowercase", () => { - const originalText = "A B "; - const { processedText, preserved } = preserveHtmlContent(originalText); - - // Simulate what happens when placeholders get converted to lowercase - const lowercasePlaceholders = processedText.toLowerCase(); - - const restoredText = restoreHtmlContent(lowercasePlaceholders, preserved); - - // Should restore the original HTML tags correctly - expect(restoredText).toBe("a b "); - expect(restoredText).toContain(""); - expect(restoredText).toContain(""); - }); - }); - - describe("sanitizeHtmlContent", () => { - it("should keep safe HTML tags", () => { - const safeHtml = - "Bold code italic"; - const sanitized = sanitizeHtmlContent(safeHtml); - expect(sanitized).toBe(safeHtml); - }); - - it("should remove unsafe HTML tags", () => { - const unsafeHtml = "Safe"; - const sanitized = sanitizeHtmlContent(unsafeHtml); - expect(sanitized).not.toContain(""); - expect(sanitized).toContain("Safe"); - }); - }); -});