Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

small fixes and upgrades

Browse Source
This commit is contained in:
Nick Leeman 2021-03-26 14:38:30 +01:00
parent cb02947a1c
commit 7b050cbfaa
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/core.ts
View File

@ -34,11 +34,12 @@ import {
Coolblue Coolblue
} from "./modules/websites"; } from "./modules/websites";
// start(); start();
debugStart(); // debugStart();
// Globals // Globals
let browser: puppeteer.Browser = null; let browser: puppeteer.Browser = null;
let sitemapUrl: string = "";
let allProducts: Array<any> = []; let allProducts: Array<any> = [];
let allBlogUrls: Array<any> = []; let allBlogUrls: Array<any> = [];
@ -54,9 +55,10 @@ async function start() {
// Get sitemap url from process params // Get sitemap url from process params
let params = process.argv.slice(2); let params = process.argv.slice(2);
sitemapUrl = params[0];
// Crawl Sitemap // Crawl Sitemap
const sitemap = await got(params[0]); const sitemap = await got(sitemapUrl);
const $ = cheerio.load(sitemap.body, { xmlMode: true }) const $ = cheerio.load(sitemap.body, { xmlMode: true })
$('loc').each(function() { $('loc').each(function() {
@ -291,7 +293,11 @@ async function crawlProductStock(url: string) {
async function generateReport() { async function generateReport() {
console.log("Generating report..."); console.log("Generating report...");
let file = `stock-report.txt`; let today: Date = new Date();
let datecode: string = today.getFullYear().toString() + (today.getMonth() + 1).toString() + today.getDay().toString();
let domain: string = getDomain(sitemapUrl);
let file = `${domain}-${datecode}-report.txt`;
let totalProducts = 0; let totalProducts = 0;
let totalOutStock = 0; let totalOutStock = 0;
for (let product of allProducts) { for (let product of allProducts) {