Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

small fixes and upgrades

Browse Source
This commit is contained in:
Nick Leeman 2021-03-26 14:38:30 +01:00
parent cb02947a1c
commit 7b050cbfaa
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/core.ts
View File

@ -34,11 +34,12 @@ import {
Coolblue
} from "./modules/websites";
// start();
debugStart();
start();
// debugStart();
// Globals
let browser: puppeteer.Browser = null;
let sitemapUrl: string = "";
let allProducts: Array<any> = [];
let allBlogUrls: Array<any> = [];
@ -54,9 +55,10 @@ async function start() {
// Get sitemap url from process params
let params = process.argv.slice(2);
sitemapUrl = params[0];
// Crawl Sitemap
const sitemap = await got(params[0]);
const sitemap = await got(sitemapUrl);
const $ = cheerio.load(sitemap.body, { xmlMode: true })
$('loc').each(function() {
@ -290,8 +292,12 @@ async function crawlProductStock(url: string) {
async function generateReport() {
console.log("Generating report...");
let file = `stock-report.txt`;
let today: Date = new Date();
let datecode: string = today.getFullYear().toString() + (today.getMonth() + 1).toString() + today.getDay().toString();
let domain: string = getDomain(sitemapUrl);
let file = `${domain}-${datecode}-report.txt`;
let totalProducts = 0;
let totalOutStock = 0;
for (let product of allProducts) {