Add @m4rc3l05/requester to replace fetch calls directly

Signed-off-by: m4rc3l05 <[email protected]>

deno.json

@@ -24,6 +24,7 @@
  "@db/sqlite": "jsr:@db/[email protected]",
  "@m4rc3l05/cron": "jsr:@m4rc3l05/[email protected]",
  "@m4rc3l05/process-lifecycle": "jsr:@m4rc3l05/[email protected]",
+ "@m4rc3l05/requester": "jsr:@m4rc3l05/[email protected]",
  "@m4rc3l05/sqlite-tag": "jsr:@m4rc3l05/[email protected]",
  "@mozilla/readability": "npm:@mozilla/[email protected]",
  "@std/async": "jsr:@std/[email protected]",

deno.lock

@@ -6,12 +6,12 @@
  "jsr:@denosaurs/plug@1": "jsr:@denosaurs/[email protected]",
  "jsr:@m4rc3l05/[email protected]": "jsr:@m4rc3l05/[email protected]",
  "jsr:@m4rc3l05/[email protected]": "jsr:@m4rc3l05/[email protected]",
+ "jsr:@m4rc3l05/[email protected]": "jsr:@m4rc3l05/[email protected]",
  "jsr:@m4rc3l05/[email protected]": "jsr:@m4rc3l05/[email protected]",
  "jsr:@std/assert@^0.217.0": "jsr:@std/[email protected]",
  "jsr:@std/assert@^0.221.0": "jsr:@std/[email protected]",
  "jsr:@std/assert@^0.224.0": "jsr:@std/[email protected]",
  "jsr:@std/[email protected]": "jsr:@std/[email protected]",
- "jsr:@std/[email protected]": "jsr:@std/[email protected]",
  "jsr:@std/bytes@^0.224.0": "jsr:@std/[email protected]",
  "jsr:@std/[email protected]": "jsr:@std/[email protected]",
  "jsr:@std/[email protected]": "jsr:@std/[email protected]",
@@ -65,6 +65,9 @@
  "@m4rc3l05/[email protected]": {
  "integrity": "f900dfe4d28e8418a0d62f7636f8a3315b7b87a142a059fb50f62aa415d7585b"
  },
+ "@m4rc3l05/[email protected]": {
+ "integrity": "f778a433273dbad03e2b98c584bee910399f85469f2009488c4e7adc6361c816"
+ },
  "@m4rc3l05/[email protected]": {
  "integrity": "3dff883763c06e97f836fe1ab82482a82e897a574072f24029ade6a5d4836fbb"
  },
@@ -80,12 +83,6 @@
  "@std/[email protected]": {
  "integrity": "61f0e9c53bf81f516aa7f54843f307eb5f05c5cf9902575a636ead162b5dfafe"
  },
- "@std/[email protected]": {
- "integrity": "b6da423eeafbd0003fe88d950e069368c6a156f8b5293b7adbd9c8903a8f8d66",
- "dependencies": [
- "jsr:@std/assert@^0.224.0"
- ]
- },
  "@std/[email protected]": {
  "integrity": "a2250e1d0eb7d1c5a426f21267ab9bdeac2447fa87a3d0d1a467d3f7a6058e49"
  },
@@ -308,6 +305,7 @@
  "jsr:@db/[email protected]",
  "jsr:@m4rc3l05/[email protected]",
  "jsr:@m4rc3l05/[email protected]",
+ "jsr:@m4rc3l05/[email protected]",
  "jsr:@m4rc3l05/[email protected]",
  "jsr:@std/[email protected]",
  "jsr:@std/[email protected]",

src/apps/api/routes/feed-items/extract-content.ts

@@ -6,6 +6,8 @@ import { DOMParser, type Element } from "deno-dom";
 import vine from "@vinejs/vine";
 import { makeLogger } from "#src/common/logger/mod.ts";
 import type { FeedItemsTable } from "#src/database/types/mod.ts";
+import { Requester } from "@m4rc3l05/requester";
+import * as requesterComposers from "@m4rc3l05/requester/composers";
 
 const requestParamsSchema = vine.object({
  id: vine.string(),
@@ -14,6 +16,9 @@ const requestParamsSchema = vine.object({
 const requestParametersValidator = vine.compile(requestParamsSchema);
 
 const log = makeLogger("extract-feed-item-contents");
+const requester = new Requester().with(
+ requesterComposers.timeout({ ms: 10_000 }),
+).build();
 
 export const extractContent = (router: Hono) => {
  router.get(
@@ -34,15 +39,12 @@ export const extractContent = (router: Hono) => {
  }
 
  try {
- const pageContent = await fetch(result?.link, {
+ const pageContent = await requester(result?.link, {
  headers: {
  "User-Agent":
  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.3",
  },
- signal: AbortSignal.any([
- AbortSignal.timeout(10_000),
- c.req.raw.signal,
- ]),
+ signal: c.req.raw.signal,
  }).then((response) => response.text());
 
  const url = new URL(result?.link);

src/apps/api/routes/feeds/validate-url.ts

@@ -15,9 +15,14 @@ export const validateUrl = (router: Hono) => {
  async (c) => {
  try {
  const data = await requestBodyValidator.validate(await c.req.json());
- const { feed: extracted, feedResolver } = await c
- .get("feedService")
- .verifyFeed(data.url, { signal: c.req.raw.signal });
+ const { feed: extracted, feedResolver } = await c.get("feedService")
+ .verifyFeed(data.url, {
+ signal: AbortSignal.any([
+ AbortSignal.timeout(10_000),
+ c.get("shutdown"),
+ c.req.raw.signal,
+ ]),
+ });
 
  const title = feedResolver.resolveFeedTitle(extracted);
 

src/apps/web/services/common/base-service.ts

@@ -1,13 +1,19 @@
 import { deepMerge } from "@std/collections";
 import { encodeBase64 } from "@std/encoding/base64";
+import { Requester } from "@m4rc3l05/requester";
+import * as requesterComposers from "@m4rc3l05/requester/composers";
 
 export abstract class BaseService {
  #baseUrl: string;
  #auth: { username: string; password: string };
+ #requester: ReturnType<Requester["build"]>;
 
  constructor(baseUrl: string, auth: { username: string; password: string }) {
  this.#baseUrl = baseUrl;
  this.#auth = auth;
+ this.#requester = new Requester().with(
+ requesterComposers.timeout({ ms: 10000 }),
+ ).build();
  }
 
  request(
@@ -17,7 +23,7 @@ export abstract class BaseService {
  sendResponse?: boolean;
  },
  ) {
- return fetch(
+ return this.#requester(
  `${this.#baseUrl}${path}`,
  // deno-lint-ignore no-explicit-any
  deepMerge((init ?? {}) as any, {
@@ -26,9 +32,6 @@ export abstract class BaseService {
  encodeBase64(`${this.#auth.username}:${this.#auth.password}`)
  }`,
  },
- signal: init?.signal
- ? AbortSignal.any([init.signal, AbortSignal.timeout(10_000)])
- : AbortSignal.timeout(10_000),
  }),
  ).then((response) => {
  if (sendResponse) return response;

src/services/feed-service.ts

@@ -1,8 +1,9 @@
 import { createHash } from "node:crypto";
 import { sql } from "@m4rc3l05/sqlite-tag";
 import * as entities from "@std/html";
+import { Requester } from "@m4rc3l05/requester";
+import * as requesterComposers from "@m4rc3l05/requester/composers";
 import * as _ from "lodash-es";
-import { request } from "#src/common/utils/fetch-utils.ts";
 import type { CustomDatabase } from "../database/mod.ts";
 import type { FeedsTable } from "../database/types/mod.ts";
 import {
@@ -44,9 +45,18 @@ const resolveFeedResolver = (contentType: string): FeedResolver => {
 
 class FeedService {
  #db: CustomDatabase;
+ #requester: ReturnType<Requester["build"]>;
 
  constructor(db: CustomDatabase) {
  this.#db = db;
+ this.#requester = new Requester().with(
+ requesterComposers.timeout({ ms: 5000 }),
+ requesterComposers.retry({
+ maxRetries: 3,
+ shouldRetry: ({ error }) =>
+ !!error && !["AbortError"].includes(error.name),
+ }),
+ ).build();
  }
 
  async syncFeed(feed: FeedsTable, options: { signal?: AbortSignal }) {
@@ -117,7 +127,7 @@ class FeedService {
  async verifyFeed(url: string, options?: { signal: AbortSignal }) {
  const { data: rawFeed, feedResolver } = await this.#extractRawFeed(
  url,
- options as Omit<typeof options, "database">,
+ options ?? {},
  );
  const parsed = feedResolver.toObject(rawFeed);
  const feed = feedResolver.resolveFeed(parsed!);
@@ -131,10 +141,10 @@ class FeedService {
 
  async #extractRawFeed(url: string, options: { signal?: AbortSignal }) {
  try {
- const response = await request(url, options, { maxRetries: 1 });
+ const response = await this.#requester(url, options);
 
  if (!response.ok) {
- throw new Error(`Error fetching feed ${url}`, {
+ throw new Error(`Request is not ok`, {
  cause: {
  response: _.pick(response, [
  "headers",
@@ -147,6 +157,10 @@ class FeedService {
  });
  }
 
+ if (response.status === 304) {
+ throw new Error("Feed did not change");
+ }
+
  if (!response.headers.has("content-type")) {
  throw new Error("No content type header in response");
  }