Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fixed csv config.

Browse Source
This commit is contained in:
Nick Leeman 2023-06-21 14:03:27 +02:00
parent 67a7b1d570
commit 3438daaa80
1 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/core.ts
View File

@ -93,7 +93,11 @@ async function start() {
{id: 'product_url', title: 'PRODUCT_URL'}, {id: 'product_url', title: 'PRODUCT_URL'},
{id: 'in_stock', title: 'IN_STOCK'}, {id: 'in_stock', title: 'IN_STOCK'},
{id: 'blog_url', title: 'BLOG_URL'} {id: 'blog_url', title: 'BLOG_URL'}
] ],
encoding: "utf8",
recordDelimiter: "\r\n",
fieldDelimiter: ",",
alwaysQuote: true,
}); });
console.log(`Initialized csv writer.`); console.log(`Initialized csv writer.`);
@ -135,7 +139,7 @@ async function crawlUrl(url: string) {
const products: Array<IProduct> = []; const products: Array<IProduct> = [];
if ($(".row-products").length >= 1) { if ($(".row-products").length >= 1) {
console.log(`- Detected ${$(".row-products").length} content egg row type products.`); console.log(`- Parsed ${$(".row-products").length} content egg row type products.`);
$(".row-products .cegg-list-logo-title a").each((index, element) => { $(".row-products .cegg-list-logo-title a").each((index, element) => {
const productUrl = $(element).attr("href"); const productUrl = $(element).attr("href");
@ -152,11 +156,11 @@ async function crawlUrl(url: string) {
} }
if ($("div.egg-container.egg-item").length >= 1) { if ($("div.egg-container.egg-item").length >= 1) {
console.log(`- Detected ${$("div.egg-container.egg-item").length} content egg large type products.`); console.log(`- Parsed ${$("div.egg-container.egg-item").length} content egg large type products.`);
$("div.egg-container.egg-item").each((index, element) => { $("div.egg-container.egg-item").each((index, element) => {
let productUrl = $(element).find(".cegg-btn-row a").first().attr("href"); const productUrl = $(element).find(".cegg-btn-row a").first().attr("href");
let name = $(element).find("h2").first().html().trim(); const name = $(element).find("h2").first().html().trim();
products.push({ products.push({
domain: "", domain: "",
@ -185,7 +189,13 @@ async function crawlUrl(url: string) {
} }
// Write to csv // Write to csv
await writer.writeRecords([products[index]]); await writer.writeRecords([{
domain: products[index].domain,
product_name: products[index].product_name,
product_url: products[index].product_url,
in_stock: products[index].in_stock,
blog_url: products[index].blog_url,
}]);
} catch (error) { } catch (error) {
console.log(` [ERROR] ${products[index].product_name} - ${products[index].domain} - ${products[index].product_url}`, error); console.log(` [ERROR] ${products[index].product_name} - ${products[index].domain} - ${products[index].product_url}`, error);
continue; continue;