added website support

2020-11-21 12:07:10 +01:00
parent 64a4ce76e2
commit 9cfda0e5f5
5 changed files with 336 additions and 264 deletions
--- a/src/core.ts
+++ b/src/core.ts
@@ -11,7 +11,11 @@ import {
    BolCom,
    BeterSport,
    FitnessGeest,
-    BodyAndFit
+    BodyAndFit,
+    PerfectBody,
+    Alternate,
+    Expert,
+    Silvergear
 } from "./modules/websites";

 start();
@@ -20,6 +24,7 @@ start();
 let browser: puppeteer.Browser = null;
 let allProducts: Array<any> = [];
 let allBlogUrls: Array<any> = [];
+let urlsFileName = "";

 async function start() {
    try {
@@ -27,16 +32,23 @@ async function start() {
        
        console.log(`Starting crawler browser..`);
        browser = await puppeteer.launch({ headless: true });
+        
+        let params = process.argv.slice(2);
+        urlsFileName = params[0].replace(/\.[^/.]+$/, "");

-        const urls = await jsonfile.readFile("./urls-check.json");
-        console.log(`Loaded ${urls.length} urls from file! (urls-check.json)`);
+        const urls = await jsonfile.readFile(`./${params[0]}`);
+        console.log(`Loaded ${urls.length} urls from file! (${params[0]})`);

        console.log(`Initialization done!`);
        console.log(`------------------------------------- \n`);

        for (let url of urls) {
-            allBlogUrls.push(url);
-            await crawlBlogPage(url);
+            try {
+                allBlogUrls.push(url);
+                await crawlBlogPage(url); 
+            } catch (error) {
+                continue;   
+            }
        }

        await browser.close();
@@ -65,37 +77,61 @@ async function crawlBlogPage(url: string) {

        let products: Array<any> = [];

-        console.log(`Detected ${$(".row-products").length} content egg products!`);
+        if ($(".row-products").length >= 1) {
+            console.log(`Detected ${$(".row-products").length} content egg row type products!`);

-        $(".row-products .cegg-list-logo-title a").each((index, element) => {
-            let productUrl = $(element).attr("href");
-            let name = $(element).html().trim();
-            
-            products.push({
-                name: name,
-                blogUrl: url,
-                url: productUrl,
-                domain: "",
-                inStock: false,
+            $(".row-products .cegg-list-logo-title a").each((index, element) => {
+                let productUrl = $(element).attr("href");
+                let name = $(element).html().trim();
+                
+                products.push({
+                    name: name,
+                    blogUrl: url,
+                    url: productUrl,
+                    domain: "",
+                    inStock: false,
+                });
            });
-        });
+        }
+        
+        if ($("div.egg-container.egg-item").length >= 1) {
+            console.log(`Detected ${$(".row-products").length} content egg large type products!`);
+
+            $("div.egg-container.egg-item").each((index, element) => {
+                let productUrl = $(element).find(".cegg-btn-row a").first().attr("href");
+                let name = $(element).find("h2").first().html().trim();
+                
+                products.push({
+                    name: name,
+                    blogUrl: url,
+                    url: productUrl,
+                    domain: "",
+                    inStock: false,
+                });
+            });
+        }

        console.log("Checking product stocks...");

        for (let index in products) {
-            let status = await crawlProductStock(products[index].url);
-            
-            products[index].domain = status[0]; 
-            products[index].url = status[1];
-            products[index].inStock = status[2];
-           
-            if (products[index].inStock) {
-                console.log(`✅ ${products[index].name} - ${products[index].domain}`);
-            } else {
-                console.log(`❌ ${products[index].name} - ${products[index].domain}`);
-            }
+            try {
+                let status = await crawlProductStock(products[index].url);

-            allProducts.push(products[index]);
+                products[index].domain = status[0]; 
+                products[index].url = status[1];
+                products[index].inStock = status[2];
+            
+                if (products[index].inStock) {
+                    console.log(`✅ ${products[index].name} - ${products[index].domain}`);
+                } else {
+                    console.log(`❌ ${products[index].name} - ${products[index].domain}`);
+                }
+
+                allProducts.push(products[index]);
+            } catch (error) {
+                console.log("Skipping product...");
+                continue;
+            }
        }

        console.log("");
@@ -142,10 +178,28 @@ async function crawlProductStock(url: string) {
            
            case "bodyandfit.com":
                return [domain, page.url(), await BodyAndFit.check(html)];
+
+            case "gorillasports.nl":
+                return [domain, page.url(), await BodyAndFit.check(html)];
+            
+            case "perfectbody.nl":
+                return [domain, page.url(), await PerfectBody.check(html)];
+            
+            case "alternate.nl":
+                return [domain, page.url(), await Alternate.check(html)];
+
+            case "expert.nl":
+                return [domain, page.url(), await Expert.check(html)];
+            
+            case "bcc.nl":
+                return [domain, page.url(), await Expert.check(html)];
+            
+            case "silvergear.eu":
+                return [domain, page.url(), await Silvergear.check(html)];
            
            default:
                console.error(`-- ${domain} is not an available website module! Can't check stock!`);
-                return ["NOT_MODULE", false];
+                return [domain, page.url(), false];
        }
    } catch (error) {
        console.error(error);
@@ -156,7 +210,7 @@ async function crawlProductStock(url: string) {
 async function generateReport() {
    console.log("Generating report...");
    
-    let file = "./report.txt";
+    let file = `${urlsFileName}-report.txt`;
    let totalProducts = 0;
    let totalOutStock = 0;
    for (let product of allProducts) {
--- a/src/modules/websites.ts
+++ b/src/modules/websites.ts
@@ -126,4 +126,127 @@ export namespace BodyAndFit {
            return false;
        }
    }
+}
+
+export namespace GorillaSports {
+    export async function check(html: string) {
+        try {
+            const $ = cheerio.load(html);
+            if ($('.product_buy .button_sold_out').length >= 1) {
+                return false;
+            } else {
+                return true;
+            }
+        } catch (error) {
+            console.log(error);
+            console.error(`Error occured during stock check!`);
+            return false;
+        }
+    }
+}
+
+export namespace PerfectBody {
+    export async function check(html: string) {
+        try {
+            const $ = cheerio.load(html);
+            if ($('.product.type-product').first().hasClass("instock")) {
+                return true;
+            } else {
+                return false;
+            }
+        } catch (error) {
+            console.log(error);
+            console.error(`Error occured during stock check!`);
+            return false;
+        }
+    }
+}
+
+export namespace BCC {
+    export async function check(html: string) {
+        try {
+            const $ = cheerio.load(html);
+            if ($(".productoffer__availability .productoffer__deliverymsg .icon").length >= 1) {
+                return true;
+            } else {
+                return false;
+            }
+        } catch (error) {
+            console.log(error);
+            console.error(`Error occured during stock check!`);
+            return false;
+        }
+    }
+}
+
+export namespace Alternate {
+    export async function check(html: string) {
+        try {
+            const $ = cheerio.load(html);
+            if ($(".productMainContainerRow .stockStatus.available_stock")) {
+                if ($(".productMainContainerRow .stockStatus.available_stock").first().html().trim().toLocaleLowerCase() >= "direct leverbaar") {
+                    return true;
+                } else {
+                    return false;
+                }
+            } else {
+                return false;
+            }
+        } catch (error) {
+            console.log(error);
+            console.error(`Error occured during stock check!`);
+            return false;
+        }
+    }
+}
+
+export namespace Expert {
+    export async function check(html: string) {
+        try {
+            const $ = cheerio.load(html);
+            if ($(".product__view .product__add-to-cart").length >= 1) {
+                return true;
+            } else {
+                return false;
+            }
+        } catch (error) {
+            console.log(error);
+            console.error(`Error occured during stock check!`);
+            return false;
+        }
+    }
+}
+
+export namespace Silvergear {
+    export async function check(html: string) {
+        try {
+            const $ = cheerio.load(html);
+            if ($('.product.type-product').first().hasClass("instock")) {
+                return true;
+            } else {
+                return false;
+            }
+        } catch (error) {
+            console.log(error);
+            console.error(`Error occured during stock check!`);
+            return false;
+        }
+    }
+}
+
+export namespace AlsaNature {
+    export async function check(html: string) {
+        try {
+            const $ = cheerio.load(html);
+            if ($('.product .inventoryStatus .icon').first().hasClass("icon-in_stock")) {
+                return true;
+            } else {
+                return false;
+            }
+        } catch (error) {
+            console.log(error);
+            console.error(`Error occured during stock check!`);
+            return false;
+        }
+    }
 }