import { singleton } from "tsyringe";
import {
  assignTransferProtocolMeta,
  RPCHost,
  RPCReflection,
  AssertionFailureError,
  assignMeta,
  RawString,
} from "civkit/civ-rpc";
import { marshalErrorLike } from "civkit/lang";
import { objHashMd5B64Of } from "civkit/hash";
import _ from "lodash";

import {
  RateLimitControl,
  RateLimitDesc,
  RateLimitTriggeredError,
} from "../shared/services/rate-limit";

import { CrawlerHost, ExtraScrappingOptions } from "./crawler";
import { CrawlerOptions, RESPOND_TIMING } from "../dto/crawler-options";
import {
  SnapshotFormatter,
  FormattedPage as RealFormattedPage,
} from "../services/snapshot-formatter";
import { GoogleSearchExplicitOperatorsDto } from "../services/serper-search";

import { GlobalLogger } from "../services/logger";
import { AsyncLocalContext } from "../services/async-context";
import { Context, Ctx, Method, Param, RPCReflect } from "../services/registry";
import { OutputServerEventStream } from "../lib/transform-server-event-stream";
import { ApiTokenAccount, AuthDTO } from "../dto/auth";

import { GoogleSERP } from "../services/serp/google";
import {
  SerperBingSearchService,
  SerperGoogleSearchService,
} from "../services/serp/serper";
import { toAsyncGenerator } from "../utils/misc";
import { LRUCache } from "lru-cache";
import { API_CALL_STATUS } from "../shared/db/api-roll";
import { SERPResult } from "../db/searched";
import { SerperSearchQueryParams } from "../shared/3rd-party/serper-search";
import { WebSearchEntry } from "../services/serp/compat";
import { SecretExposer } from "../shared/services/secrets";
import { StandaloneSearchFallbackService } from "../services/serp/standalone-fallback";
import { STANDALONE_BOOT_TIMEOUT_MS } from "../services/boot-timeouts";

interface FormattedPage extends RealFormattedPage {
  favicon?: string;
  date?: string;
}

type RateLimitCache = {
  blockedUntil?: Date;
  user?: ApiTokenAccount;
};

@singleton()
export class SearcherHost extends RPCHost {
  logger = this.globalLogger.child({ service: this.constructor.name });

  cacheRetentionMs = 1000 * 3600 * 24 * 7;
  cacheValidMs = 1000 * 3600;
  pageCacheToleranceMs = 1000 * 3600 * 24;

  reasonableDelayMs = 8_000;
  standaloneSearchPageTimeoutMs = 12_000;

  targetResultCount = 5;

  highFreqKeyCache = new LRUCache<string, RateLimitCache>({
    max: 256,
    ttl: 60 * 60 * 1000,
    updateAgeOnGet: false,
    updateAgeOnHas: false,
  });

  batchedCaches: SERPResult[] = [];

  constructor(
    protected globalLogger: GlobalLogger,
    protected rateLimitControl: RateLimitControl,
    protected threadLocal: AsyncLocalContext,
    protected secretExposer: SecretExposer,
    protected crawler: CrawlerHost,
    protected snapshotFormatter: SnapshotFormatter,
    protected googleSerp: GoogleSERP,
    protected standaloneFallback: StandaloneSearchFallbackService,
    protected serperGoogle: SerperGoogleSearchService,
    protected serperBing: SerperBingSearchService,
  ) {
    super(...arguments);

    setInterval(
      () => {
        const thisBatch = this.batchedCaches;
        this.batchedCaches = [];
        if (!thisBatch.length) {
          return;
        }
        const batch = SERPResult.DB.batch();

        for (const x of thisBatch) {
          batch.set(SERPResult.COLLECTION.doc(), x.degradeForFireStore());
        }
        batch
          .commit()
          .then(() => {
            this.logger.debug(`Saved ${thisBatch.length} caches by batch`);
          })
          .catch((err) => {
            this.logger.warn(`Failed to cache search result in batch`, { err });
          });
      },
      1000 * 10 + Math.round(1000 * Math.random()),
    ).unref();
  }

  override async init() {
    await this.dependencyReady(STANDALONE_BOOT_TIMEOUT_MS);

    this.emit("ready");
  }

  override dependencyReady(timeoutMilliseconds = STANDALONE_BOOT_TIMEOUT_MS) {
    return super.dependencyReady(timeoutMilliseconds);
  }

  @Method({
    name: "searchIndex",
    ext: {
      http: {
        action: ["get", "post"],
        path: "/search",
      },
    },
    tags: ["search"],
    returnType: [String, OutputServerEventStream],
  })
  @Method({
    ext: {
      http: {
        action: ["get", "post"],
        path: "::q",
      },
    },
    tags: ["search"],
    returnType: [String, OutputServerEventStream, RawString],
  })
  async search(
    @RPCReflect() rpcReflect: RPCReflection,
    @Ctx() ctx: Context,
    auth: AuthDTO,
    crawlerOptions: CrawlerOptions,
    searchExplicitOperators: GoogleSearchExplicitOperatorsDto,
    @Param("count", { validate: (v: number) => v >= 0 && v <= 20 })
    count: number,
    @Param("type", { type: new Set(["web", "images", "news"]), default: "web" })
    variant: "web" | "images" | "news",
    @Param("provider", { type: new Set(["google", "bing"]), default: "google" })
    searchEngine: "google" | "bing",
    @Param("num", { validate: (v: number) => v >= 0 && v <= 20 })
    num?: number,
    @Param("gl", { validate: (v: string) => /^[a-z]{2}$/i.test(v || "") })
    gl?: string,
    @Param("hl", { validate: (v: string) => /^[a-z]{2}$/i.test(v || "") })
    hl?: string,
    @Param("location") location?: string,
    @Param("page") page?: number,
    @Param("fallback", { type: Boolean, default: true }) fallback?: boolean,
    @Param("q") q?: string,
  ) {
    // We want to make our search API follow SERP schema, so we need to expose 'num' parameter.
    // Since we used 'count' as 'num' previously, we need to keep 'count' for old users.
    // Here we combine 'count' and 'num' to 'count' for the rest of the function.
    count = (num !== undefined ? num : count) ?? 10;

    const authToken = auth.bearerToken;
    let highFreqKey: RateLimitCache | undefined;
    if (authToken && this.highFreqKeyCache.has(authToken)) {
      highFreqKey = this.highFreqKeyCache.get(authToken)!;
      auth.user = highFreqKey.user;
      auth.uid = highFreqKey.user?.user_id;
    }

    const uid = await auth.solveUID();
    // Return content by default
    const crawlWithoutContent =
      crawlerOptions.respondWith.includes("no-content");
    const withFavicon = Boolean(ctx.get("X-With-Favicons"));
    this.threadLocal.set("collect-favicon", withFavicon);
    crawlerOptions.respondTiming ??= RESPOND_TIMING.VISIBLE_CONTENT;

    let chargeAmount = 0;
    const noSlashPath = decodeURIComponent(ctx.path).slice(1);
    if (!noSlashPath && !q) {
      const index = await this.crawler.getIndex(ctx, auth);
      if (
        !ctx.accepts("text/plain") &&
        (ctx.accepts("text/json") || ctx.accepts("application/json"))
      ) {
        return index;
      }

      return assignTransferProtocolMeta(`${index}`, {
        contentType: "text/plain",
        envelope: null,
      });
    }

    const user = await auth.assertUser();

    if (highFreqKey?.blockedUntil) {
      const now = new Date();
      const blockedTimeRemaining =
        highFreqKey.blockedUntil.valueOf() - now.valueOf();
      if (blockedTimeRemaining > 0) {
        throw RateLimitTriggeredError.from({
          message: `Per UID rate limit exceeded (async)`,
          retryAfter: Math.ceil(blockedTimeRemaining / 1000),
        });
      }
    }

    const rateLimitPolicy = auth.getRateLimits(
      rpcReflect.name.toUpperCase(),
    ) || [
      parseInt(user.metadata?.speed_level) >= 2
        ? RateLimitDesc.from({
            occurrence: 1000,
            periodSeconds: 60,
          })
        : RateLimitDesc.from({
            occurrence: 100,
            periodSeconds: 60,
          }),
    ];

    const apiRollPromise = uid
      ? this.rateLimitControl.simpleRPCUidBasedLimit(
          rpcReflect,
          uid,
          [rpcReflect.name.toUpperCase()],
          ...rateLimitPolicy,
        )
      : this.rateLimitControl.simpleRpcIPBasedLimit(
          rpcReflect,
          ctx.ip || "anonymous",
          [rpcReflect.name.toUpperCase()],
          [[new Date(Date.now() - 60 * 1000), 20]],
        );

    if (uid && !highFreqKey) {
      // Normal path
      await apiRollPromise;

      if (
        rateLimitPolicy.some((x) => {
          const rpm = x.occurrence / (x.periodSeconds / 60);
          if (rpm >= 400) {
            return true;
          }

          return false;
        })
      ) {
        this.highFreqKeyCache.set(auth.bearerToken!, {
          user,
        });
      }
    } else if (uid && highFreqKey) {
      // High freq key path
      apiRollPromise
        .then(
          // Rate limit not triggered, make sure not blocking.
          () => {
            delete highFreqKey.blockedUntil;
          },
          // Rate limit triggered
          (err) => {
            if (!(err instanceof RateLimitTriggeredError)) {
              return;
            }
            const now = Date.now();
            let tgtDate;
            if (err.retryAfterDate) {
              tgtDate = err.retryAfterDate;
            } else if (err.retryAfter) {
              tgtDate = new Date(now + err.retryAfter * 1000);
            }

            if (tgtDate) {
              const dt = tgtDate.valueOf() - now;
              highFreqKey.blockedUntil = tgtDate;
              setTimeout(() => {
                if (highFreqKey.blockedUntil === tgtDate) {
                  delete highFreqKey.blockedUntil;
                }
              }, dt).unref();
            }
          },
        )
        .finally(async () => {
          // Always asynchronously update user(wallet);
          const user = await auth.getBrief().catch(() => undefined);
          if (user) {
            highFreqKey.user = user;
          }
        });
    }

    rpcReflect.finally(async () => {
      if (chargeAmount) {
        if (uid) {
          auth
            .reportUsage(chargeAmount, `xread-${rpcReflect.name}`)
            .catch((err) => {
              this.logger.warn(`Unable to report usage for ${uid}`, {
                err: marshalErrorLike(err),
              });
            });
        }
        try {
          const apiRoll = await apiRollPromise;
          apiRoll.chargeAmount = chargeAmount;
          await apiRoll.save({ merge: true });
        } catch (_err) {
          await this.rateLimitControl
            .record({
              uid,
              tags: [rpcReflect.name.toUpperCase()],
              status: API_CALL_STATUS.SUCCESS,
              chargeAmount,
            })
            .save()
            .catch((err) => {
              this.logger.warn(`Failed to save rate limit record`, {
                err: marshalErrorLike(err),
              });
            });
        }
      }
    });

    delete crawlerOptions.html;

    const crawlOpts = await this.crawler.configure(crawlerOptions);
    const searchQuery = searchExplicitOperators.addTo(q || noSlashPath);

    let fetchNum = count;
    if ((page ?? 1) === 1) {
      fetchNum = count > 10 ? 30 : 20;
    }

    let fallbackQuery: string | undefined;
    let chargeAmountScaler = 1;
    if (searchEngine === "bing") {
      this.threadLocal.set("bing-preferred", true);
      chargeAmountScaler = 3;
    }

    if (variant !== "web") {
      chargeAmountScaler = 5;
    }

    // Search with fallback logic if enabled
    const searchParams = {
      variant,
      provider: searchEngine,
      q: searchQuery,
      num: fetchNum,
      gl,
      hl,
      location,
      page,
    };

    const {
      results,
      query: successQuery,
      tryTimes,
    } = await this.searchWithFallback(
      searchParams,
      fallback,
      crawlerOptions.noCache,
    );
    chargeAmountScaler *= tryTimes;

    fallbackQuery = successQuery !== searchQuery ? successQuery : undefined;

    if (!results.length) {
      throw new AssertionFailureError(
        `No search results available for query ${searchQuery}`,
      );
    }

    crawlOpts.timeoutMs = Math.min(
      crawlOpts.timeoutMs || this.standaloneSearchPageTimeoutMs,
      this.standaloneSearchPageTimeoutMs,
    );

    let lastScrapped: any[] | undefined;
    const targetResultCount = crawlWithoutContent ? count : count + 2;
    const trimmedResults: any[] = results
      .filter((x) => Boolean(x.link))
      .slice(0, targetResultCount)
      .map((x) => this.mapToFinalResults(x));
    trimmedResults.toString = function () {
      let r =
        this.map((x, i) => (x ? Reflect.apply(x.toString, x, [i]) : ""))
          .join("\n\n")
          .trimEnd() + "\n";
      if (fallbackQuery) {
        r = `Fallback query: ${fallbackQuery}\n\n${r}`;
      }
      return r;
    };
    if (
      !crawlerOptions.respondWith.includes("no-content") &&
      ["html", "text", "shot", "markdown", "content"].some((x) =>
        crawlerOptions.respondWith.includes(x),
      )
    ) {
      for (const x of trimmedResults) {
        x.content ??= "";
      }
    }
    const assigningOfGeneralMixins = Promise.allSettled(
      trimmedResults.map((x) => this.assignGeneralMixin(x)),
    );

    let it;

    if (crawlWithoutContent || count === 0) {
      it = toAsyncGenerator(trimmedResults);
      await assigningOfGeneralMixins;
    } else {
      it = this.fetchSearchResults(
        crawlerOptions.respondWith,
        trimmedResults,
        crawlOpts,
        CrawlerOptions.from({
          ...crawlerOptions,
          cacheTolerance:
            crawlerOptions.cacheTolerance ?? this.pageCacheToleranceMs,
        }),
        count,
      );
    }

    if (!ctx.accepts("text/plain") && ctx.accepts("text/event-stream")) {
      const sseStream = new OutputServerEventStream();
      rpcReflect.return(sseStream);
      try {
        for await (const scrapped of it) {
          if (!scrapped) {
            continue;
          }
          if (rpcReflect.signal.aborted) {
            break;
          }

          chargeAmount = this.assignChargeAmount(
            scrapped,
            count,
            chargeAmountScaler,
            fallbackQuery,
          );
          lastScrapped = scrapped;

          if (fallbackQuery) {
            sseStream.write({
              event: "meta",
              data: { fallback: fallbackQuery },
            });
          }

          sseStream.write({
            event: "data",
            data: scrapped,
          });
        }
      } catch (err: any) {
        this.logger.error(
          `Failed to collect search result for query ${searchQuery}`,
          { err: marshalErrorLike(err) },
        );
        sseStream.write({
          event: "error",
          data: marshalErrorLike(err),
        });
      }

      sseStream.end();

      return sseStream;
    }

    let earlyReturn = false;
    if (
      !ctx.accepts("text/plain") &&
      (ctx.accepts("text/json") || ctx.accepts("application/json"))
    ) {
      let earlyReturnTimer: ReturnType<typeof setTimeout> | undefined;
      const setEarlyReturnTimer = () => {
        if (earlyReturnTimer) {
          return;
        }
        earlyReturnTimer = setTimeout(
          async () => {
            if (!lastScrapped) {
              return;
            }
            await assigningOfGeneralMixins;
            chargeAmount = this.assignChargeAmount(
              lastScrapped,
              count,
              chargeAmountScaler,
              fallbackQuery,
            );

            rpcReflect.return(lastScrapped);
            earlyReturn = true;
          },
          (crawlerOptions.timeout || 0) * 1000 || this.reasonableDelayMs,
        );
      };

      for await (const scrapped of it) {
        lastScrapped = scrapped;
        if (rpcReflect.signal.aborted || earlyReturn) {
          break;
        }
        if (_.some(scrapped, (x) => this.pageQualified(x))) {
          setEarlyReturnTimer();
        }
        if (!this.searchResultsQualified(scrapped, count)) {
          continue;
        }
        if (earlyReturnTimer) {
          clearTimeout(earlyReturnTimer);
        }
        await assigningOfGeneralMixins;
        chargeAmount = this.assignChargeAmount(
          scrapped,
          count,
          chargeAmountScaler,
          fallbackQuery,
        );

        return scrapped;
      }

      if (earlyReturnTimer) {
        clearTimeout(earlyReturnTimer);
      }

      if (!lastScrapped) {
        throw new AssertionFailureError(
          `No content available for query ${searchQuery}`,
        );
      }

      if (!earlyReturn) {
        await assigningOfGeneralMixins;
        chargeAmount = this.assignChargeAmount(
          lastScrapped,
          count,
          chargeAmountScaler,
          fallbackQuery,
        );
      }

      return lastScrapped;
    }

    let earlyReturnTimer: ReturnType<typeof setTimeout> | undefined;
    const setEarlyReturnTimer = () => {
      if (earlyReturnTimer) {
        return;
      }
      earlyReturnTimer = setTimeout(
        async () => {
          if (!lastScrapped) {
            return;
          }
          await assigningOfGeneralMixins;
          chargeAmount = this.assignChargeAmount(
            lastScrapped,
            count,
            chargeAmountScaler,
            fallbackQuery,
          );

          rpcReflect.return(
            assignTransferProtocolMeta(`${lastScrapped}`, {
              contentType: "text/plain",
              envelope: null,
            }),
          );
          earlyReturn = true;
        },
        (crawlerOptions.timeout || 0) * 1000 || this.reasonableDelayMs,
      );
    };

    for await (const scrapped of it) {
      lastScrapped = scrapped;
      if (rpcReflect.signal.aborted || earlyReturn) {
        break;
      }
      if (_.some(scrapped, (x) => this.pageQualified(x))) {
        setEarlyReturnTimer();
      }

      if (!this.searchResultsQualified(scrapped, count)) {
        continue;
      }

      if (earlyReturnTimer) {
        clearTimeout(earlyReturnTimer);
      }
      await assigningOfGeneralMixins;
      chargeAmount = this.assignChargeAmount(
        scrapped,
        count,
        chargeAmountScaler,
        fallbackQuery,
      );

      return assignTransferProtocolMeta(`${scrapped}`, {
        contentType: "text/plain",
        envelope: null,
      });
    }

    if (earlyReturnTimer) {
      clearTimeout(earlyReturnTimer);
    }

    if (!lastScrapped) {
      throw new AssertionFailureError(
        `No content available for query ${searchQuery}`,
      );
    }

    if (!earlyReturn) {
      await assigningOfGeneralMixins;
      chargeAmount = this.assignChargeAmount(
        lastScrapped,
        count,
        chargeAmountScaler,
        fallbackQuery,
      );
    }

    return assignTransferProtocolMeta(`${lastScrapped}`, {
      contentType: "text/plain",
      envelope: null,
    });
  }

  /**
   * Search with fallback to progressively shorter queries if no results found
   * @param params Search parameters
   * @param useFallback Whether to use the fallback mechanism
   * @param noCache Whether to bypass cache
   * @returns Search response and the successful query
   */
  async searchWithFallback(
    params: SerperSearchQueryParams & {
      variant: "web" | "images" | "news";
      provider?: string;
    },
    useFallback: boolean = false,
    noCache: boolean = false,
  ) {
    // Try original query first
    const originalQuery = params.q;
    const containsRTL =
      /[\u0600-\u06FF\u0750-\u077F\u08A0-\u08FF\uFB50-\uFDFF\uFE70-\uFEFF\u0590-\u05FF\uFB1D-\uFB4F\u0700-\u074F\u0780-\u07BF\u07C0-\u07FF]/.test(
        originalQuery,
      );

    // Extract results based on variant
    let tryTimes = 1;
    const results = await this.cachedSearch(params.variant, params, noCache);
    if (results.length || !useFallback) {
      return { results, query: params.q, tryTimes };
    }

    let queryTerms = originalQuery.split(/\s+/);
    const lastResort = containsRTL
      ? queryTerms.slice(queryTerms.length - 2)
      : queryTerms.slice(0, 2);

    this.logger.info(
      `No results for "${originalQuery}", trying fallback queries`,
    );

    let terms: string[] = [];
    // fallback n times
    const n = 4;

    while (tryTimes < n) {
      const delta = Math.ceil(queryTerms.length / n) * tryTimes;
      terms = containsRTL
        ? queryTerms.slice(delta)
        : queryTerms.slice(0, queryTerms.length - delta);
      const query = terms.join(" ");
      if (!query) {
        break;
      }
      tryTimes += 1;
      this.logger.info(`Retrying search with fallback query: "${query}"`);
      const fallbackParams = { ...params, q: query };
      const fallbackResults = await this.cachedSearch(
        params.variant,
        fallbackParams,
        noCache,
      );
      if (fallbackResults.length > 0) {
        return { results: fallbackResults, query: fallbackParams.q, tryTimes };
      }
    }

    if (terms.length > lastResort.length) {
      const query = lastResort.join(" ");
      this.logger.info(`Retrying search with fallback query: "${query}"`);
      const fallbackParams = { ...params, q: query };
      tryTimes += 1;
      const fallbackResults = await this.cachedSearch(
        params.variant,
        fallbackParams,
        noCache,
      );

      if (fallbackResults.length > 0) {
        return { results: fallbackResults, query, tryTimes };
      }
    }

    return { results, query: originalQuery, tryTimes };
  }

  async *fetchSearchResults(
    mode:
      | string
      | "markdown"
      | "html"
      | "text"
      | "screenshot"
      | "favicon"
      | "content",
    searchResults?: FormattedPage[],
    options?: ExtraScrappingOptions,
    crawlerOptions?: CrawlerOptions,
    count?: number,
  ) {
    if (!searchResults) {
      return;
    }
    const urls = searchResults.map((x) => new URL(x.url!));
    const snapshotMap = new WeakMap();
    for await (const scrapped of this.crawler.scrapMany(
      urls,
      options,
      crawlerOptions,
    )) {
      const mapped = scrapped.map((x, i) => {
        if (!x) {
          return {};
        }
        if (snapshotMap.has(x)) {
          return snapshotMap.get(x);
        }
        return this.crawler
          .formatSnapshotWithPDFSideLoad(mode, x, urls[i], undefined, options)
          .then((r) => {
            snapshotMap.set(x, r);

            return r;
          })
          .catch((err) => {
            this.logger.error(`Failed to format snapshot for ${urls[i].href}`, {
              err: marshalErrorLike(err),
            });

            return {};
          });
      });

      const resultArray = (await Promise.all(mapped)) as FormattedPage[];
      for (const [i, v] of resultArray.entries()) {
        if (v) {
          Object.assign(searchResults[i], v);
        }
      }

      yield this.reOrganizeSearchResults(searchResults, count);
    }
  }

  reOrganizeSearchResults(searchResults: FormattedPage[], count?: number) {
    const targetResultCount = count || this.targetResultCount;
    const [qualifiedPages, unqualifiedPages] = _.partition(searchResults, (x) =>
      this.pageQualified(x),
    );
    const acceptSet = new Set(qualifiedPages);

    const n = targetResultCount - qualifiedPages.length;
    for (const x of unqualifiedPages.slice(0, n >= 0 ? n : 0)) {
      acceptSet.add(x);
    }

    const filtered = searchResults
      .filter((x) => acceptSet.has(x))
      .slice(0, targetResultCount);

    const resultArray = filtered;

    resultArray.toString = searchResults.toString;

    return resultArray;
  }

  assignChargeAmount(
    formatted: FormattedPage[],
    num: number,
    scaler: number,
    fallbackQuery?: string,
  ) {
    let contentCharge = 0;
    for (const x of formatted) {
      const itemAmount = this.crawler.assignChargeAmount(x) || 0;

      if (!itemAmount) {
        continue;
      }

      contentCharge += itemAmount;
    }

    const numCharge = Math.ceil(formatted.length / 10) * 10000 * scaler;

    const final = Math.max(contentCharge, numCharge);

    if (final === numCharge) {
      for (const x of formatted) {
        x.usage = { tokens: Math.ceil(numCharge / formatted.length) };
      }
    }

    const metadata: Record<string, any> = { usage: { tokens: final } };
    if (fallbackQuery) {
      metadata.fallback = fallbackQuery;
    }

    assignMeta(formatted, metadata);

    return final;
  }

  pageQualified(formattedPage: FormattedPage) {
    return (
      (formattedPage.title && formattedPage.content) ||
      formattedPage.screenshotUrl ||
      formattedPage.pageshotUrl ||
      formattedPage.text ||
      formattedPage.html
    );
  }

  searchResultsQualified(
    results: FormattedPage[],
    targetResultCount = this.targetResultCount,
  ) {
    return (
      _.every(results, (x) => this.pageQualified(x)) &&
      results.length >= targetResultCount
    );
  }

  async getFavicon(domain: string) {
    const url = `https://www.google.com/s2/favicons?sz=32&domain_url=${domain}`;

    try {
      const response = await fetch(url);
      if (!response.ok) {
        return "";
      }
      const ab = await response.arrayBuffer();
      const buffer = Buffer.from(ab);
      const base64 = buffer.toString("base64");
      return `data:image/png;base64,${base64}`;
    } catch (error: any) {
      this.logger.warn(`Failed to get favicon base64 string`, {
        err: marshalErrorLike(error),
      });
      return "";
    }
  }

  *iterProviders(preference?: string, _variant?: string) {
    const preferScraping = !this.secretExposer.SERPER_SEARCH_API_KEY;
    const includeSerper = Boolean(this.secretExposer.SERPER_SEARCH_API_KEY);

    if (preference === "bing") {
      if (preferScraping) {
        yield this.standaloneFallback;
      }
      if (includeSerper) {
        yield this.serperBing;
        yield this.serperGoogle;
      }
      yield this.googleSerp;
      if (includeSerper) {
        yield this.serperGoogle;
      }

      return;
    }

    if (preference === "google") {
      if (preferScraping) {
        yield this.standaloneFallback;
        yield this.googleSerp;
      }
      yield this.googleSerp;
      if (includeSerper) {
        yield this.serperGoogle;
        yield this.serperGoogle;
      }

      return;
    }

    if (preferScraping) {
      yield this.standaloneFallback;
      yield this.googleSerp;
    }
    yield this.googleSerp;
    if (includeSerper) {
      yield this.serperGoogle;
      yield this.serperGoogle;
      yield this.serperBing;
    }
  }

  async cachedSearch(
    variant: "web" | "news" | "images",
    query: Record<string, any>,
    noCache?: boolean,
  ): Promise<WebSearchEntry[]> {
    const queryDigest = objHashMd5B64Of({ ...query, variant });
    const provider = query.provider;
    Reflect.deleteProperty(query, "provider");
    let cache;
    if (!noCache) {
      cache = (
        await SERPResult.fromFirestoreQuery(
          SERPResult.COLLECTION.where("queryDigest", "==", queryDigest)
            .orderBy("createdAt", "desc")
            .limit(1),
        )
      )[0];
      if (cache) {
        const age = Date.now() - cache.createdAt.valueOf();
        const stale =
          cache.createdAt.valueOf() < Date.now() - this.cacheValidMs;
        this.logger.info(
          `${stale ? "Stale cache exists" : "Cache hit"} for search query "${query.q}", normalized digest: ${queryDigest}, ${age}ms old`,
          {
            query,
            digest: queryDigest,
            age,
            stale,
          },
        );

        if (!stale) {
          return cache.response as any;
        }
      }
    }

    try {
      let r: any[] | undefined;
      let lastError;
      outerLoop: for (const client of this.iterProviders(provider, variant)) {
        const t0 = Date.now();
        try {
          switch (variant) {
            case "images": {
              r = await Reflect.apply(client.imageSearch, client, [query]);
              break;
            }
            case "news": {
              r = await Reflect.apply(client.newsSearch, client, [query]);
              break;
            }
            case "web":
            default: {
              r = await Reflect.apply(client.webSearch, client, [query]);
              break;
            }
          }
          const dt = Date.now() - t0;
          this.logger.info(
            `Search took ${dt}ms, ${client.constructor.name}(${variant})`,
            { searchDt: dt, variant, client: client.constructor.name },
          );
          break outerLoop;
        } catch (err) {
          lastError = err;
          const dt = Date.now() - t0;
          this.logger.warn(
            `Failed to do ${variant} search using ${client.constructor.name}`,
            { err, variant, searchDt: dt },
          );
        }
      }

      if (r?.length) {
        const nowDate = new Date();
        const record = SERPResult.from({
          query,
          queryDigest,
          response: r,
          createdAt: nowDate,
          expireAt: new Date(nowDate.valueOf() + this.cacheRetentionMs),
        });

        this.batchedCaches.push(record);
      } else if (lastError) {
        throw lastError;
      }

      return r as WebSearchEntry[];
    } catch (err: any) {
      if (cache) {
        this.logger.warn(
          `Failed to fetch search result, but a stale cache is available. falling back to stale cache`,
          { err: marshalErrorLike(err) },
        );

        return cache.response as any;
      }

      throw err;
    }
  }

  mapToFinalResults(input: WebSearchEntry) {
    const whitelistedProps = [
      "imageUrl",
      "imageWidth",
      "imageHeight",
      "source",
      "date",
      "siteLinks",
    ];
    const result = {
      title: input.title,
      url: input.link,
      description: Reflect.get(input, "snippet"),
      ..._.pick(input, whitelistedProps),
    };

    return result;
  }

  async assignGeneralMixin(result: FormattedPage) {
    const collectFavicon = this.threadLocal.get("collect-favicon");

    if (collectFavicon && result.url) {
      const url = new URL(result.url);
      Reflect.set(result, "favicon", await this.getFavicon(url.origin));
    }

    Object.setPrototypeOf(result, searchResultProto);
  }
}

const dataItems = [
  { key: "title", label: "Title" },
  { key: "source", label: "Source" },
  { key: "url", label: "URL Source" },
  { key: "imageUrl", label: "Image URL" },
  { key: "description", label: "Description" },
  { key: "publishedTime", label: "Published Time" },
  { key: "imageWidth", label: "Image Width" },
  { key: "imageHeight", label: "Image Height" },
  { key: "date", label: "Date" },
  { key: "favicon", label: "Favicon" },
];

const searchResultProto = {
  toString(this: FormattedPage, i?: number) {
    const chunks = [];
    for (const item of dataItems) {
      const v = Reflect.get(this, item.key);
      if (typeof v !== "undefined") {
        if (i === undefined) {
          chunks.push(`[${item.label}]: ${v}`);
        } else {
          chunks.push(`[${i + 1}] ${item.label}: ${v}`);
        }
      }
    }

    if (this.content) {
      chunks.push(`\n${this.content}`);
    }

    if (this.images) {
      const imageSummaryChunks = [
        `${i === undefined ? "" : `[${i + 1}] `}Images:`,
      ];
      for (const [k, v] of Object.entries(this.images)) {
        imageSummaryChunks.push(`- ![${k}](${v})`);
      }
      if (imageSummaryChunks.length === 1) {
        imageSummaryChunks.push(
          "This page does not seem to contain any images.",
        );
      }
      chunks.push(imageSummaryChunks.join("\n"));
    }
    if (this.links) {
      const linkSummaryChunks = [
        `${i === undefined ? "" : `[${i + 1}] `}Links/Buttons:`,
      ];
      if (Array.isArray(this.links)) {
        for (const [k, v] of this.links) {
          linkSummaryChunks.push(`- [${k}](${v})`);
        }
      } else {
        for (const [k, v] of Object.entries(this.links)) {
          linkSummaryChunks.push(`- [${k}](${v})`);
        }
      }
      if (linkSummaryChunks.length === 1) {
        linkSummaryChunks.push(
          "This page does not seem to contain any buttons/links.",
        );
      }
      chunks.push(linkSummaryChunks.join("\n"));
    }

    return chunks.join("\n");
  },
};