Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

added basic support.

Browse Source
This commit is contained in:
Nick Leeman 2023-06-21 13:08:55 +02:00
parent 3b485e4da7
commit 0fa05020e3
2 changed files with 57 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/core.ts
View File

@ -5,7 +5,7 @@ import fs from "fs";
import { getDomain } from "tldts";
import { Browser, launch } from "puppeteer";
import axios from "axios";
import { BolCom } from "./modules/websites";
import { Dobey, LDJsonParser } from "./modules/websites";
interface IProduct {
name: string;
@ -164,7 +164,7 @@ async function crawlUrl(url: string) {
if (products[index].inStock) {
console.log(` [IN STOCK] ${products[index].name} - ${products[index].domain}`);
} else {
console.log(` [OUT OF STOCK] ${products[index].name} - ${products[index].domain}`);
console.log(` [OUT OF STOCK] ${products[index].name} - ${products[index].domain} - ${products[index].url}`);
}
allProducts.push(products[index]);
@ -182,6 +182,8 @@ async function crawlUrl(url: string) {
}
}
// https://pornhub.com/view_video.php?viewkey=ph5f0b0b2b2b2a4
async function crawlProductStock(url: string) {
try {
// Open new page and goto url
@ -202,7 +204,22 @@ async function crawlProductStock(url: string) {
switch (domain) {
case "bol.com":
return [domain, page.url(), await BolCom.check(html)];
return [domain, page.url(), await LDJsonParser.check(html)];
case "petsplace.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
case "dobey.nl":
return [domain, page.url(), await Dobey.check(html)];
case "brekz.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
case "hondenbed.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
case "petsonline.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
default:
console.error(`-- ${domain} is not an supported website! Cannot check stock!`);

37
src/modules/websites.ts
View File

@ -14,8 +14,7 @@ export namespace Template {
}
}
export namespace BolCom {
export namespace LDJsonParser {
export async function check(html: string) {
try {
const $ = cheerio.load(html);
@ -25,13 +24,28 @@ export namespace BolCom {
let json = JSON.parse($(element).html());
if (json["@type"]) {
if (json["@type"] == "Product") {
if (json["@type"].includes("Product")) {
if (Array.isArray(json["offers"])) {
let anyStock = false;
for (let index in json["offers"]) {
if (json["offers"][index]["availability"].includes("InStock")) {
anyStock = true;
}
}
if (anyStock) {
stock = true;
return;
}
} else {
if (json["offers"]["availability"].includes("InStock")) {
stock = true;
return;
}
}
}
}
});
return stock;
@ -44,3 +58,20 @@ export namespace BolCom {
}
export namespace Dobey {
export async function check(html: string) {
try {
const $ = cheerio.load(html);
if ($("#product_view #stock_indicator").hasClass("stock_green")) {
return true;
} else {
return false;
}
} catch (error) {
console.log(error);
console.error(`Error occured during stock check!`);
return false;
}
}
}