Frozenverse/ContentEggStockCrawler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fixed csv config.

Browse Source
This commit is contained in:
Nick Leeman 2023-06-21 14:03:27 +02:00
parent 67a7b1d570
commit 3438daaa80
1 changed files with 17 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
src/core.ts
View File

@ -93,7 +93,11 @@ async function start() {
{id: 'product_url', title: 'PRODUCT_URL'},
{id: 'in_stock', title: 'IN_STOCK'},
{id: 'blog_url', title: 'BLOG_URL'}
]
],
encoding: "utf8",
recordDelimiter: "\r\n",
fieldDelimiter: ",",
alwaysQuote: true,
});
console.log(`Initialized csv writer.`);
@ -135,7 +139,7 @@ async function crawlUrl(url: string) {
const products: Array<IProduct> = [];
if ($(".row-products").length >= 1) {
console.log(`- Detected ${$(".row-products").length} content egg row type products.`);
console.log(`- Parsed ${$(".row-products").length} content egg row type products.`);
$(".row-products .cegg-list-logo-title a").each((index, element) => {
const productUrl = $(element).attr("href");
@ -152,11 +156,11 @@ async function crawlUrl(url: string) {
}
if ($("div.egg-container.egg-item").length >= 1) {
console.log(`- Detected ${$("div.egg-container.egg-item").length} content egg large type products.`);
console.log(`- Parsed ${$("div.egg-container.egg-item").length} content egg large type products.`);
$("div.egg-container.egg-item").each((index, element) => {
let productUrl = $(element).find(".cegg-btn-row a").first().attr("href");
let name = $(element).find("h2").first().html().trim();
const productUrl = $(element).find(".cegg-btn-row a").first().attr("href");
const name = $(element).find("h2").first().html().trim();
products.push({
domain: "",
@ -183,9 +187,15 @@ async function crawlUrl(url: string) {
} else {
console.log(` [OUT OF STOCK] ${products[index].product_name} - ${products[index].domain}`);
}
// Write to csv
await writer.writeRecords([products[index]]);
await writer.writeRecords([{
domain: products[index].domain,
product_name: products[index].product_name,
product_url: products[index].product_url,
in_stock: products[index].in_stock,
blog_url: products[index].blog_url,
}]);
} catch (error) {
console.log(` [ERROR] ${products[index].product_name} - ${products[index].domain} - ${products[index].product_url}`, error);
continue;