Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

added dectdirect support

Browse Source
This commit is contained in:
Nick Leeman 2023-07-16 12:08:51 +02:00
parent 84edf517d5
commit f34af7e673
2 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/cli.ts
View File

@ -5,7 +5,7 @@ import fs from "fs";
import { getDomain } from "tldts"; import { getDomain } from "tldts";
import { Browser, launch } from "puppeteer"; import { Browser, launch } from "puppeteer";
import axios from "axios"; import axios from "axios";
import { CoolBlue, Dobey, EP, LDJsonParser, Macrovet, Mediamarkt } from "./modules/websites"; import { CoolBlue, Dectdirect, Dobey, EP, LDJsonParser, Macrovet, Mediamarkt } from "./modules/websites";
import { createObjectCsvWriter } from "csv-writer"; import { createObjectCsvWriter } from "csv-writer";
import { CsvWriter } from "csv-writer/src/lib/csv-writer"; import { CsvWriter } from "csv-writer/src/lib/csv-writer";
import { ObjectMap } from "csv-writer/src/lib/lang/object"; import { ObjectMap } from "csv-writer/src/lib/lang/object";
@ -28,8 +28,8 @@ start();
async function debugStart() { async function debugStart() {
browser = await launch({ headless: false }); browser = await launch({ headless: false });
console.log(await crawlProductStock("https://www.mediamarkt.nl/nl/product/_sitecom-ca-040-usb-c-naar-usb-c-gamingkabel-met-pd-1658247.html")); console.log(await crawlProductStock("https://www.dectdirect.nl/nl/unifi-switch-lite-8-poe-115502613.html"));
console.log(await crawlProductStock("https://www.mediamarkt.nl/nl/product/_apple-iphone-14-128gb-midnight-1738479.html")); // console.log(await crawlProductStock("https://www.mediamarkt.nl/nl/product/_apple-iphone-14-128gb-midnight-1738479.html"));
// console.log(await crawlProductStock("https://www.coolblue.nl/product/923036/hp-deskjet-2720e-all-in-one.html?clickref=1101lwW9ebAE&utm_source=performancehorizon&utm_medium=affiliate&utm_campaign=Comparison%2FReview&utm_content=1101lwW9ebAE&utm_term=1100leWsm&ref=293530&PHGref=1101lwW9ebAE&cmt=c_ph%2Capm_Comparison%2FReview_%2Cacid1101l93%2Cacr_1100leWsm%2Caclr_1101lwW9ebAE")); // console.log(await crawlProductStock("https://www.coolblue.nl/product/923036/hp-deskjet-2720e-all-in-one.html?clickref=1101lwW9ebAE&utm_source=performancehorizon&utm_medium=affiliate&utm_campaign=Comparison%2FReview&utm_content=1101lwW9ebAE&utm_term=1100leWsm&ref=293530&PHGref=1101lwW9ebAE&cmt=c_ph%2Capm_Comparison%2FReview_%2Cacid1101l93%2Cacr_1100leWsm%2Caclr_1101lwW9ebAE"));
// console.log(await crawlProductStock("https://www.coolblue.nl/product/882996/canon-pixma-ts-3450-zwart.html?clickref=1011lwWDBFEL&utm_source=performancehorizon&utm_medium=affiliate&utm_campaign=Comparison%2FReview&utm_content=1011lwWDBFEL&utm_term=1100leWsm&ref=293530&PHGref=1011lwWDBFEL&cmt=c_ph%2Capm_Comparison%2FReview_%2Cacid1101l93%2Cacr_1100leWsm%2Caclr_1011lwWDBFEL")); // console.log(await crawlProductStock("https://www.coolblue.nl/product/882996/canon-pixma-ts-3450-zwart.html?clickref=1011lwWDBFEL&utm_source=performancehorizon&utm_medium=affiliate&utm_campaign=Comparison%2FReview&utm_content=1011lwWDBFEL&utm_term=1100leWsm&ref=293530&PHGref=1011lwWDBFEL&cmt=c_ph%2Capm_Comparison%2FReview_%2Cacid1101l93%2Cacr_1100leWsm%2Caclr_1011lwWDBFEL"));
} }
@ -282,6 +282,9 @@ async function crawlProductStock(url: string): Promise<[string, string, boolean]
case "mediamarkt.nl": case "mediamarkt.nl":
return [domain, page.url(), await Mediamarkt.check(html)]; return [domain, page.url(), await Mediamarkt.check(html)];
case "dectdirect.nl":
return [domain, page.url(), await Dectdirect.check(html)];
default: default:
console.error(`-- ${domain} is not an supported website! Cannot check stock!`); console.error(`-- ${domain} is not an supported website! Cannot check stock!`);
return [domain, page.url(), false]; return [domain, page.url(), false];

22
src/modules/websites.ts
View File

@ -121,6 +121,28 @@ export namespace Mediamarkt {
} }
} }
export namespace Dectdirect {
export async function check(html: string) {
try {
const $ = cheerio.load(html);
// Check if .rounded-full .bg-green-500 class exists with text content op voorraad
$('.js-product .rounded-full').each((index, element) => {
if ($(element).text().includes("op voorraad")) {
return true;
}
});
return false;
} catch (error) {
console.log(error);
console.error(`Error occured during stock check!`);
return false;
}
}
}
export namespace Dobey { export namespace Dobey {
export async function check(html: string) { export async function check(html: string) {
try { try {