diff --git a/src/cli.ts b/src/cli.ts index 1632872..47b3c8a 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5,7 +5,7 @@ import fs from "fs"; import { getDomain } from "tldts"; import { Browser, launch } from "puppeteer"; import axios from "axios"; -import { CoolBlue, Dobey, EP, LDJsonParser, Macrovet } from "./modules/websites"; +import { CoolBlue, Dobey, EP, LDJsonParser, Macrovet, Mediamarkt } from "./modules/websites"; import { createObjectCsvWriter } from "csv-writer"; import { CsvWriter } from "csv-writer/src/lib/csv-writer"; import { ObjectMap } from "csv-writer/src/lib/lang/object"; @@ -23,12 +23,13 @@ let browser: Browser = null; let sitemapUrl: string = ""; let writer: CsvWriter> = null; -// debugStart(); -start(); +debugStart(); +// start(); async function debugStart() { browser = await launch({ headless: false }); - console.log(await crawlProductStock("https://prf.hn/click/camref:1100leWsm/destination:https%3A%2F%2Fwww.coolblue.nl%2Fproduct%2F856992%2Fmio-cyclo-discover.html")); + console.log(await crawlProductStock("https://www.mediamarkt.nl/nl/product/_sitecom-ca-040-usb-c-naar-usb-c-gamingkabel-met-pd-1658247.html")); + console.log(await crawlProductStock("https://www.mediamarkt.nl/nl/product/_apple-iphone-14-128gb-midnight-1738479.html")); // console.log(await crawlProductStock("https://www.coolblue.nl/product/923036/hp-deskjet-2720e-all-in-one.html?clickref=1101lwW9ebAE&utm_source=performancehorizon&utm_medium=affiliate&utm_campaign=Comparison%2FReview&utm_content=1101lwW9ebAE&utm_term=1100leWsm&ref=293530&PHGref=1101lwW9ebAE&cmt=c_ph%2Capm_Comparison%2FReview_%2Cacid1101l93%2Cacr_1100leWsm%2Caclr_1101lwW9ebAE")); // console.log(await crawlProductStock("https://www.coolblue.nl/product/882996/canon-pixma-ts-3450-zwart.html?clickref=1011lwWDBFEL&utm_source=performancehorizon&utm_medium=affiliate&utm_campaign=Comparison%2FReview&utm_content=1011lwWDBFEL&utm_term=1100leWsm&ref=293530&PHGref=1011lwWDBFEL&cmt=c_ph%2Capm_Comparison%2FReview_%2Cacid1101l93%2Cacr_1100leWsm%2Caclr_1011lwWDBFEL")); } @@ -278,6 +279,9 @@ async function crawlProductStock(url: string): Promise<[string, string, boolean] case "ezydog.nl": return [domain, page.url(), await LDJsonParser.check(html)]; + case "mediamarkt.nl": + return [domain, page.url(), await Mediamarkt.check(html)]; + default: console.error(`-- ${domain} is not an supported website! Cannot check stock!`); return [domain, page.url(), false]; diff --git a/src/modules/websites.ts b/src/modules/websites.ts index 8f2b295..ce6fe4e 100644 --- a/src/modules/websites.ts +++ b/src/modules/websites.ts @@ -20,8 +20,14 @@ export namespace LDJsonParser { const snippets: Array = []; const $ = cheerio.load(html); $('script[type="application/ld+json"]').each((index, element) => { - snippets.push($(element).html()); - + let html = $(element).html(); + // remove all spaces + html = html.replace(/\s/g, ''); + + // remove all newlines + html = html.replace(/\n/g, ''); + + snippets.push(html); }); for (const snippet of snippets) { @@ -64,13 +70,20 @@ export namespace CoolBlue { const snippets: Array = []; const $ = cheerio.load(html); $('script[type="application/ld+json"]').each((index, element) => { - snippets.push($(element).html()); + let html = $(element).html(); + // remove all spaces + html = html.replace(/\s/g, ''); + + // remove all newlines + html = html.replace(/\n/g, ''); + + snippets.push(html); }); for (const snippet of snippets) { try { - if (snippet.includes(`"@type": "Product"`)) { - if (snippet.includes(`"availability": "https://schema.org/InStock"`)) { + if (snippet.includes(`"@type":"Product"`)) { + if (snippet.includes(`"availability":"http://schema.org/InStock"`) || snippet.includes(`"availability":"https://schema.org/InStock"`)) { return true; } } @@ -89,6 +102,24 @@ export namespace CoolBlue { } } +export namespace Mediamarkt { + export async function check(html: string) { + try { + const $ = cheerio.load(html); + + // Check if data-test="mms-delivery-online-availability_AVAILABLE" exists + if ($('[data-test="mms-delivery-online-availability_AVAILABLE"]').length > 0) { + return true; + } + + return false; + } catch (error) { + console.log(error); + console.error(`Error occured during stock check!`); + return false; + } + } +} export namespace Dobey { export async function check(html: string) {