Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

added basic support.

Browse Source
This commit is contained in:
Nick Leeman 2023-06-21 13:08:55 +02:00
parent 3b485e4da7
commit 0fa05020e3
2 changed files with 57 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/core.ts
View File

@ -5,7 +5,7 @@ import fs from "fs";
import { getDomain } from "tldts"; import { getDomain } from "tldts";
import { Browser, launch } from "puppeteer"; import { Browser, launch } from "puppeteer";
import axios from "axios"; import axios from "axios";
import { BolCom } from "./modules/websites"; import { Dobey, LDJsonParser } from "./modules/websites";
interface IProduct { interface IProduct {
name: string; name: string;
@ -164,7 +164,7 @@ async function crawlUrl(url: string) {
if (products[index].inStock) { if (products[index].inStock) {
console.log(` [IN STOCK] ${products[index].name} - ${products[index].domain}`); console.log(` [IN STOCK] ${products[index].name} - ${products[index].domain}`);
} else { } else {
console.log(` [OUT OF STOCK] ${products[index].name} - ${products[index].domain}`); console.log(` [OUT OF STOCK] ${products[index].name} - ${products[index].domain} - ${products[index].url}`);
} }
allProducts.push(products[index]); allProducts.push(products[index]);
@ -182,6 +182,8 @@ async function crawlUrl(url: string) {
} }
} }
// https://pornhub.com/view_video.php?viewkey=ph5f0b0b2b2b2a4
async function crawlProductStock(url: string) { async function crawlProductStock(url: string) {
try { try {
// Open new page and goto url // Open new page and goto url
@ -202,7 +204,22 @@ async function crawlProductStock(url: string) {
switch (domain) { switch (domain) {
case "bol.com": case "bol.com":
return [domain, page.url(), await BolCom.check(html)]; return [domain, page.url(), await LDJsonParser.check(html)];
case "petsplace.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
case "dobey.nl":
return [domain, page.url(), await Dobey.check(html)];
case "brekz.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
case "hondenbed.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
case "petsonline.nl":
return [domain, page.url(), await LDJsonParser.check(html)];
default: default:
console.error(`-- ${domain} is not an supported website! Cannot check stock!`); console.error(`-- ${domain} is not an supported website! Cannot check stock!`);

43
src/modules/websites.ts
View File

@ -14,8 +14,7 @@ export namespace Template {
} }
} }
export namespace LDJsonParser {
export namespace BolCom {
export async function check(html: string) { export async function check(html: string) {
try { try {
const $ = cheerio.load(html); const $ = cheerio.load(html);
@ -25,10 +24,25 @@ export namespace BolCom {
let json = JSON.parse($(element).html()); let json = JSON.parse($(element).html());
if (json["@type"]) { if (json["@type"]) {
if (json["@type"] == "Product") { if (json["@type"].includes("Product")) {
if (json["offers"]["availability"].includes("InStock")) { if (Array.isArray(json["offers"])) {
stock = true; let anyStock = false;
return;
for (let index in json["offers"]) {
if (json["offers"][index]["availability"].includes("InStock")) {
anyStock = true;
}
}
if (anyStock) {
stock = true;
return;
}
} else {
if (json["offers"]["availability"].includes("InStock")) {
stock = true;
return;
}
} }
} }
} }
@ -44,3 +58,20 @@ export namespace BolCom {
} }
export namespace Dobey {
export async function check(html: string) {
try {
const $ = cheerio.load(html);
if ($("#product_view #stock_indicator").hasClass("stock_green")) {
return true;
} else {
return false;
}
} catch (error) {
console.log(error);
console.error(`Error occured during stock check!`);
return false;
}
}
}