diff --git a/deno.json b/deno.json index fd8eb8c..9a6398c 100644 --- a/deno.json +++ b/deno.json @@ -5,7 +5,6 @@ "lint": "deno lint ./src/", "test": "deno test --coverage=.coverage", "coverage": "deno coverage .coverage --lcov --exclude=/src/node/ --exclude=/test/ --exclude=/scripts/ > ./.coverage/coverage.lcov", - "test:watch": "deno test --watch", "update:version": "deno run --allow-read --allow-write ./scripts/update-version.ts", "update:deno_deps": "deno run -A https://deno.land/x/udd/main.ts dev_deps.ts ./scripts/build-npm.ts ./src/lib/Pastebin.ts ./src/lib/Scraper.ts", diff --git a/src/lib/Scraper.ts b/src/lib/Scraper.ts index 8ab0c6a..6423ace 100644 --- a/src/lib/Scraper.ts +++ b/src/lib/Scraper.ts @@ -105,7 +105,7 @@ const convertData = (paste: ScrapePasteRaw): ScrapePaste => { export class Scraper { // We're able to overwrite fetch because it is an abstract class - fetch = globalThis.fetch; + #fetch = globalThis.fetch; #events = new Event(); @@ -139,7 +139,7 @@ export class Scraper { }; if (fetch) { - this.fetch = fetch; + this.#fetch = fetch; } if ( @@ -226,7 +226,7 @@ export class Scraper { const realKey = key .replace("https://scrape.pastebin.com/api_scrape_item.php?i=", "") .replace("https://pastebin.com/", ""); - return this.fetch(`${ENDPOINTS.RAW}${realKey}`) + return this.#fetch(`${ENDPOINTS.RAW}${realKey}`) .then((response) => { if (!response.ok) { throw new Error(`Error fetching raw: ${response.statusText}`); @@ -245,7 +245,7 @@ export class Scraper { const realKey = key .replace("https://scrape.pastebin.com/api_scrape_item.php?i=", "") .replace("https://pastebin.com/", ""); - return this.fetch(`${ENDPOINTS.META}${realKey}`) + return this.#fetch(`${ENDPOINTS.META}${realKey}`) .then((response) => { if (!response.ok) { throw new Error(`Error fetching meta: ${response.statusText}`); @@ -283,7 +283,7 @@ export class Scraper { } } const limited = typeof limit !== "undefined" ? limit : this.#limit; - const response = await this.fetch(`${ENDPOINTS.SCRAPE}?limit=${limited}`); + const response = await this.#fetch(`${ENDPOINTS.SCRAPE}?limit=${limited}`); if (response.status === 403) { const text = await response.text(); throw new Error(`Pastebin blocked you, have you whitelisted your IP? \n\n Response: ${text}`); @@ -339,4 +339,8 @@ export class Scraper { this.#events.post.call(this.#events, callArgs as never); } + + get fetch(): typeof globalThis.fetch { + return this.#fetch; + } } diff --git a/test/Scraper.test.ts b/test/Scraper.test.ts index 7ab4ca1..2da71a0 100644 --- a/test/Scraper.test.ts +++ b/test/Scraper.test.ts @@ -37,6 +37,14 @@ const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); Deno.test({ name: "Scraper", fn: async (context) => { + await runTest(context, "constructor", () => { + // overwrite fetch + const fetch = () => {}; + + const scraper = new Scraper({}, fetch as unknown as typeof globalThis.fetch); + assertEquals(scraper.fetch, fetch as unknown as typeof globalThis.fetch); + }); + await runTest(context, "errors", () => { assertThrows(() => new Scraper({ intervalTime: 0, limit: 0 })); assertThrows(() => new Scraper({ intervalTime: 1000, limit: 0 })); @@ -48,6 +56,14 @@ Deno.test({ mf.intercept("https://scrape.pastebin.com/api_scraping.php?limit=1").response(json(1)).times( 2, ); + mf.intercept("https://scrape.pastebin.com/api_scraping.php?limit=1").response( + JSON.stringify([ + { + ...simpleScrape, + expire: "1111111111", + }, + ]), + ); const scraper = new Scraper({}); @@ -56,6 +72,7 @@ Deno.test({ assertEquals(d.key, "abc"); assertEquals(d.title, "abc"); assertEquals(d.user, "abc"); + assertEquals(d.expire, null); assertEquals(d.size, 123); assertEquals(d.hits, 123); assertEquals(d.syntax, "abc"); @@ -70,6 +87,11 @@ Deno.test({ assertEquals(single[0].hits, 123); assertEquals(single[0].syntax, "abc"); + const second = await scraper.singleScrape(1); + + assertEquals(second.length, 1); + assertEquals(typeof second[0].expire, "object"); + mf.deactivate(); }); @@ -114,6 +136,33 @@ Deno.test({ mf.deactivate(); }); + await runTest(context, "scrape error test", async () => { + const mf = new MockFetch(); + mf.intercept("https://scrape.pastebin.com/api_scraping.php?limit=1").response( + "test", + { status: 403 }, + ); + + const started = await new Promise((resolve) => { + const scraper = new Scraper({ + limit: 1, + breakOnError: true, + }); + + scraper.on("error", async () => { + await wait(1000).then(() => { + resolve(true); + }); + }); + + scraper.start(); + }); + + assertEquals(started, true); + + mf.deactivate(); + }); + await runTest(context, "scrape raw", async () => { const mf = new MockFetch();