Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

updated small fix

Browse Source
This commit is contained in:
Nick Leeman 2023-07-16 11:02:29 +02:00
parent 421155d3fa
commit 84edf517d5
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/cli.ts
View File

@ -23,8 +23,8 @@ let browser: Browser = null;
let sitemapUrl: string = ""; let sitemapUrl: string = "";
let writer: CsvWriter<ObjectMap<any>> = null; let writer: CsvWriter<ObjectMap<any>> = null;
debugStart(); // debugStart();
// start(); start();
async function debugStart() { async function debugStart() {
browser = await launch({ headless: false }); browser = await launch({ headless: false });
@ -72,7 +72,7 @@ async function start() {
for (const sitemapUrl of sitemapUrls) { for (const sitemapUrl of sitemapUrls) {
console.log(`Crawling found sitemap: ${sitemapUrl}`); console.log(`Crawling sitemap: ${sitemapUrl}`);
const sitemapRequest = await axios.get(sitemapUrl); const sitemapRequest = await axios.get(sitemapUrl);
const $ = cheerio.load(sitemapRequest.data, { xmlMode: true }); const $ = cheerio.load(sitemapRequest.data, { xmlMode: true });
@ -85,13 +85,13 @@ async function start() {
}); });
} }
console.log(`Crawled ${urls.length} urls from sitemaps.`); console.log(`Found ${urls.length} urls from sitemaps.`);
browser = await launch({ headless: true }); browser = await launch({ headless: true });
console.log(`Initialized puppeteer browser.`); console.log(`Initialized puppeteer browser.`);
const today: Date = new Date(); const today: Date = new Date();
const datecode: string = today.getFullYear().toString() + "-" + (today.getMonth() + 1).toString() + "-" + today.getDay().toString(); const datecode: string = `${today.getFullYear()}-${today.getMonth() + 1}-${today.getDate()}`;
const domain: string = getDomain(sitemapUrl); const domain: string = getDomain(sitemapUrl);
const file = `${domain}_${datecode}_report.csv`; const file = `${domain}_${datecode}_report.csv`;
writer = createObjectCsvWriter({ writer = createObjectCsvWriter({