added report system

src/core.ts

@@ -1,6 +1,7 @@
-import puppeteer from "puppeteer";
+import puppeteer, { product } from "puppeteer";
 import jsonfile from "jsonfile";
 import cheerio from "cheerio";
+import fs from "fs-extra";
 import { getDomain } from "tldts";
 
 // Import website modules
@@ -13,6 +14,8 @@ start();
 
 // Globals
 let browser: puppeteer.Browser = null;
+let allProducts: Array<any> = [];
+let allBlogUrls: Array<any> = [];
 
 async function start() {
     try {
@@ -28,9 +31,12 @@ async function start() {
         console.log(`------------------------------------- \n`);
 
         for (let url of urls) {
+            allBlogUrls.push(url);
             await crawlBlogPage(url);
         }
 
+        await browser.close();
+        await generateReport();
     } catch (error) {
         console.error(`An Error Occured!`, error);    
     }
@@ -58,13 +64,13 @@ async function crawlBlogPage(url: string) {
         console.log(`Detected ${$(".row-products").length} content egg products!`);
 
         $(".row-products .cegg-list-logo-title a").each((index, element) => {
-            let url = $(element).attr("href");
+            let productUrl = $(element).attr("href");
             let name = $(element).html().trim();
             
             products.push({
                 name: name,
                 blogUrl: url,
-                url: "",
+                url: productUrl,
                 domain: "",
                 inStock: false,
             });
@@ -73,7 +79,7 @@ async function crawlBlogPage(url: string) {
         console.log("Checking product stocks...");
 
         for (let index in products) {
-            let status = await crawlProductStock(products[index].blogUrl);
+            let status = await crawlProductStock(products[index].url);
             
             products[index].domain = status[0]; 
             products[index].url = status[1];
@@ -84,6 +90,8 @@ async function crawlBlogPage(url: string) {
             } else {
                 console.log(`❌ ${products[index].name} - ${products[index].domain}`);
             }
+
+            allProducts.push(products[index]);
         }
 
     } catch (error) {
@@ -125,5 +133,44 @@ async function crawlProductStock(url: string) {
         console.log("Error while trying to crawl page! Skipping...");
     }
 }
-    
+
+async function generateReport() {
+    console.log("Generating report...");
+    
+    let file = "./report.txt";
+    let totalProducts = 0;
+    let totalOutStock = 0;
+    for (let product of allProducts) {
+        totalProducts++;
+
+        if (product.inStock == false) {
+            totalOutStock++;
+        }
+    }
+
+    await fs.appendFile(file, `----------------------------------------\n`);
+    await fs.appendFile(file, `Content Egg Product Stock Crawler Report\n`);
+    await fs.appendFile(file, `Total Blog Urls: ${allBlogUrls.length}\n`);
+    await fs.appendFile(file, `Total Products: ${totalProducts}\n`);
+    await fs.appendFile(file, `Total Out of Stock: ${totalOutStock}\n`);
+    await fs.appendFile(file, `----------------------------------------\n`);
+    await fs.appendFile(file, ``);
+
+    let lastBlogUrl = "";
+    for (let product of allProducts) {
+        if (product.blogUrl != lastBlogUrl) {
+            await fs.appendFile(file, `\n\n`);
+            await fs.appendFile(file, `${product.blogUrl}\n`);
+            await fs.appendFile(file, `----------------------------------\n`);
+        }
+
+        if (product.inStock == false) {
+            await fs.appendFile(file, `${product.domain} - ${product.name} - ${product.url}\n`);
+        }
+
+        lastBlogUrl = product.blogUrl;
+    }
+
+    console.log("Report generated!");
+}
 

src/modules/websites.ts

@@ -25,6 +25,7 @@ export namespace FitnessKoerier {
                 return true;
             }
         } catch (error) {
+            console.log(error);
             console.error(`Error occured during stock check!`);
             return false;
         }
@@ -36,7 +37,11 @@ export namespace FitnessApparaat {
     export async function check(html: string) {
         try {
             const $ = cheerio.load(html);
-            console.log($('[itemprop="offers"').find(".stock-red").length);
+            if ($('[itemprop="offers"]').find(".stock-red").length >= 1) {
+                return false;
+            } else {
+                return true;
+            }
         } catch (error) {
             console.log(error);
             console.error(`Error occured during stock check!`);