314 lines
11 KiB
TypeScript
314 lines
11 KiB
TypeScript
import { singleton } from 'tsyringe';
|
|
import { AsyncService } from 'civkit/async-service';
|
|
import { GlobalLogger } from '../logger';
|
|
import { JSDomControl } from '../jsdom';
|
|
import { isMainThread } from 'worker_threads';
|
|
import _ from 'lodash';
|
|
import { WebSearchEntry } from './compat';
|
|
import { ScrappingOptions, SERPSpecializedPuppeteerControl } from './puppeteer';
|
|
import { CurlControl } from '../curl';
|
|
import { readFile } from 'fs/promises';
|
|
import { ApplicationError } from 'civkit/civ-rpc';
|
|
import { ServiceBadApproachError, ServiceBadAttemptError } from '../errors';
|
|
import { parseJSONText } from 'civkit/vectorize';
|
|
import { retryWith } from 'civkit/decorators';
|
|
import { ProxyProviderService } from '../../shared/services/proxy-provider';
|
|
|
|
@singleton()
|
|
export class GoogleSERP extends AsyncService {
|
|
logger = this.globalLogger.child({ service: this.constructor.name });
|
|
googleDomain = process.env.OVERRIDE_GOOGLE_DOMAIN || 'www.google.com';
|
|
|
|
constructor(
|
|
protected globalLogger: GlobalLogger,
|
|
protected puppeteerControl: SERPSpecializedPuppeteerControl,
|
|
protected jsDomControl: JSDomControl,
|
|
protected curlControl: CurlControl,
|
|
protected proxyProvider: ProxyProviderService,
|
|
) {
|
|
const filteredDeps = isMainThread ? arguments : _.without(arguments, puppeteerControl);
|
|
super(...filteredDeps);
|
|
}
|
|
|
|
override async init() {
|
|
await this.dependencyReady();
|
|
|
|
this.emit('ready');
|
|
}
|
|
|
|
@retryWith((err) => {
|
|
if (err instanceof ServiceBadApproachError) {
|
|
return false;
|
|
}
|
|
if (err instanceof ServiceBadAttemptError) {
|
|
// Keep trying
|
|
return true;
|
|
}
|
|
if (err instanceof ApplicationError) {
|
|
// Quit with this error
|
|
return false;
|
|
}
|
|
return undefined;
|
|
}, 3)
|
|
async sideLoadWithAllocatedProxy(url: URL, opts?: ScrappingOptions) {
|
|
if (opts?.allocProxy === 'none') {
|
|
return this.curlControl.sideLoad(url, opts);
|
|
}
|
|
|
|
if (!this.proxyProvider.isConfigured()) {
|
|
this.logger.info(`No proxy configured, falling back to direct Google request`, { url: url.href });
|
|
return this.curlControl.sideLoad(url, opts);
|
|
}
|
|
|
|
const proxy = await this.proxyProvider.alloc(
|
|
process.env.PREFERRED_PROXY_COUNTRY || 'auto'
|
|
);
|
|
this.logger.debug(`Proxy allocated`, { proxy: proxy.href });
|
|
const r = await this.curlControl.sideLoad(url, {
|
|
...opts,
|
|
proxyUrl: proxy.href,
|
|
});
|
|
|
|
if (r.status === 429) {
|
|
throw new ServiceBadAttemptError('Google returned a 429 error. This may happen due to various reasons, including rate limiting or other issues.');
|
|
}
|
|
|
|
if (opts && opts.allocProxy) {
|
|
opts.proxyUrl ??= proxy.href;
|
|
}
|
|
|
|
return { ...r, proxy };
|
|
}
|
|
|
|
digestQuery(query: { [k: string]: any; }) {
|
|
const url = new URL(`https://${this.googleDomain}/search`);
|
|
const clone = { ...query };
|
|
const num = clone.num || 10;
|
|
if (clone.page) {
|
|
const page = parseInt(clone.page);
|
|
delete clone.page;
|
|
clone.start = (page - 1) * num;
|
|
if (clone.start === 0) {
|
|
delete clone.start;
|
|
}
|
|
}
|
|
if (clone.location) {
|
|
delete clone.location;
|
|
}
|
|
|
|
for (const [k, v] of Object.entries(clone)) {
|
|
if (v === undefined || v === null) {
|
|
continue;
|
|
}
|
|
url.searchParams.set(k, `${v}`);
|
|
}
|
|
|
|
return url;
|
|
}
|
|
|
|
async webSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) {
|
|
const url = this.digestQuery(query);
|
|
|
|
const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts);
|
|
if (opts && sideLoaded.sideLoadOpts) {
|
|
opts.sideLoad = sideLoaded.sideLoadOpts;
|
|
}
|
|
|
|
const snapshot = await this.puppeteerControl.controlledScrap(url, getWebSearchResults, opts);
|
|
|
|
return snapshot;
|
|
}
|
|
|
|
async newsSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) {
|
|
const url = this.digestQuery(query);
|
|
|
|
url.searchParams.set('tbm', 'nws');
|
|
|
|
const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts);
|
|
if (opts && sideLoaded.sideLoadOpts) {
|
|
opts.sideLoad = sideLoaded.sideLoadOpts;
|
|
}
|
|
|
|
const snapshot = await this.puppeteerControl.controlledScrap(url, getNewsSearchResults, opts);
|
|
|
|
return snapshot;
|
|
}
|
|
|
|
async imageSearch(query: { [k: string]: any; }, opts?: ScrappingOptions) {
|
|
const url = this.digestQuery(query);
|
|
|
|
url.searchParams.set('tbm', 'isch');
|
|
url.searchParams.set('asearch', 'isch');
|
|
url.searchParams.set('async', `_fmt:json,p:1,ijn:${query.start ? Math.floor(query.start / (query.num || 10)) : 0}`);
|
|
|
|
const sideLoaded = await this.sideLoadWithAllocatedProxy(url, opts);
|
|
|
|
if (sideLoaded.status !== 200 || !sideLoaded.file) {
|
|
throw new ServiceBadAttemptError('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.');
|
|
}
|
|
|
|
const jsonTxt = (await readFile((await sideLoaded.file.filePath))).toString();
|
|
const rJSON = parseJSONText(jsonTxt.slice(jsonTxt.indexOf('{"ischj":')));
|
|
|
|
return _.get(rJSON, 'ischj.metadata').map((x: any) => {
|
|
|
|
return {
|
|
link: _.get(x, 'result.referrer_url'),
|
|
title: _.get(x, 'result.page_title'),
|
|
snippet: _.get(x, 'text_in_grid.snippet'),
|
|
source: _.get(x, 'result.site_title'),
|
|
imageWidth: _.get(x, 'original_image.width'),
|
|
imageHeight: _.get(x, 'original_image.height'),
|
|
imageUrl: _.get(x, 'original_image.url'),
|
|
variant: 'images',
|
|
};
|
|
}) as WebSearchEntry[];
|
|
}
|
|
}
|
|
|
|
async function getWebSearchResults() {
|
|
if (location.pathname.startsWith('/sorry') || location.pathname.startsWith('/error')) {
|
|
throw new Error('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.');
|
|
}
|
|
|
|
// @ts-ignore
|
|
await Promise.race([window.waitForSelector('div[data-async-context^="query"]'), window.waitForSelector('#botstuff .mnr-c')]);
|
|
|
|
const wrapper1 = document.querySelector('div[data-async-context^="query"]');
|
|
|
|
if (!wrapper1) {
|
|
return undefined;
|
|
}
|
|
|
|
const query = decodeURIComponent(wrapper1.getAttribute('data-async-context')?.split('query:')[1] || '');
|
|
|
|
if (!query) {
|
|
return undefined;
|
|
}
|
|
|
|
const candidates = Array.from(wrapper1.querySelectorAll('div[lang],div[data-surl]'));
|
|
|
|
return candidates.map((x) => {
|
|
const primaryLink = x.querySelector('a:not([href="#"])');
|
|
if (!primaryLink) {
|
|
return undefined;
|
|
}
|
|
const url = primaryLink.getAttribute('href');
|
|
|
|
if (primaryLink.querySelector('div[role="heading"]')) {
|
|
// const spans = primaryLink.querySelectorAll('span');
|
|
// const title = spans[0]?.textContent;
|
|
// const source = spans[1]?.textContent;
|
|
// const date = spans[spans.length - 1].textContent;
|
|
|
|
// return {
|
|
// link: url,
|
|
// title,
|
|
// source,
|
|
// date,
|
|
// variant: 'video'
|
|
// };
|
|
return undefined;
|
|
}
|
|
|
|
const title = primaryLink.querySelector('h3')?.textContent;
|
|
const source = Array.from(primaryLink.querySelectorAll('span')).find((x) => x.textContent)?.textContent;
|
|
const cite = primaryLink.querySelector('cite[role=text]')?.textContent;
|
|
let date = cite?.split('·')[1]?.trim();
|
|
const snippets = Array.from(x.querySelectorAll('div[data-sncf*="1"] span'));
|
|
let snippet = snippets[snippets.length - 1]?.textContent;
|
|
if (!snippet) {
|
|
snippet = x.querySelector('div.IsZvec')?.textContent?.trim() || null;
|
|
}
|
|
date ??= snippets[snippets.length - 2]?.textContent?.trim();
|
|
const imageUrl = x.querySelector('div[data-sncf*="1"] img[src]:not(img[src^="data"])')?.getAttribute('src');
|
|
let siteLinks = Array.from(x.querySelectorAll('div[data-sncf*="3"] a[href]')).map((l) => {
|
|
return {
|
|
link: l.getAttribute('href'),
|
|
title: l.textContent,
|
|
};
|
|
});
|
|
const perhapsParent = x.parentElement?.closest('div[data-hveid]');
|
|
if (!siteLinks?.length && perhapsParent) {
|
|
const candidates = Array.from(perhapsParent.querySelectorAll('td h3'));
|
|
if (candidates.length) {
|
|
siteLinks = candidates.map((l) => {
|
|
const link = l.querySelector('a');
|
|
if (!link) {
|
|
return undefined;
|
|
}
|
|
const snippet = l.nextElementSibling?.textContent;
|
|
return {
|
|
link: link.getAttribute('href'),
|
|
title: link.textContent,
|
|
snippet,
|
|
};
|
|
}).filter(Boolean) as any;
|
|
}
|
|
}
|
|
|
|
return {
|
|
link: url,
|
|
title,
|
|
source,
|
|
date,
|
|
snippet: snippet ?? undefined,
|
|
imageUrl: imageUrl?.startsWith('data:') ? undefined : imageUrl,
|
|
siteLinks: siteLinks.length ? siteLinks : undefined,
|
|
variant: 'web',
|
|
};
|
|
}).filter(Boolean) as WebSearchEntry[];
|
|
}
|
|
|
|
async function getNewsSearchResults() {
|
|
if (location.pathname.startsWith('/sorry') || location.pathname.startsWith('/error')) {
|
|
throw new Error('Google returned an error page. This may happen due to various reasons, including rate limiting or other issues.');
|
|
}
|
|
|
|
// @ts-ignore
|
|
await Promise.race([window.waitForSelector('div[data-async-context^="query"]'), window.waitForSelector('#botstuff .mnr-c')]);
|
|
|
|
const wrapper1 = document.querySelector('div[data-async-context^="query"]');
|
|
|
|
if (!wrapper1) {
|
|
return undefined;
|
|
}
|
|
|
|
const query = decodeURIComponent(wrapper1.getAttribute('data-async-context')?.split('query:')[1] || '');
|
|
|
|
if (!query) {
|
|
return undefined;
|
|
}
|
|
|
|
const candidates = Array.from(wrapper1.querySelectorAll('div[data-news-doc-id]'));
|
|
|
|
return candidates.map((x) => {
|
|
const primaryLink = x.querySelector('a:not([href="#"])');
|
|
if (!primaryLink) {
|
|
return undefined;
|
|
}
|
|
const url = primaryLink.getAttribute('href');
|
|
const titleElem = primaryLink.querySelector('div[role="heading"]');
|
|
|
|
if (!titleElem) {
|
|
return undefined;
|
|
}
|
|
|
|
const title = titleElem.textContent?.trim();
|
|
const source = titleElem.previousElementSibling?.textContent?.trim();
|
|
const snippet = titleElem.nextElementSibling?.textContent?.trim();
|
|
|
|
const innerSpans = Array.from(titleElem.parentElement?.querySelectorAll('span') || []);
|
|
const date = innerSpans[innerSpans.length - 1]?.textContent?.trim();
|
|
|
|
return {
|
|
link: url,
|
|
title,
|
|
source,
|
|
date,
|
|
snippet,
|
|
variant: 'news',
|
|
};
|
|
}).filter(Boolean) as WebSearchEntry[];
|
|
}
|