added sitemap parser!
This commit is contained in:
Nick Leeman
parent
22aa81f2ee
commit
48fee9b86d
175
package-lock.json
generated
175
package-lock.json
generated
@ -4,6 +4,30 @@
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"@sindresorhus/is": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.0.tgz",
|
||||
"integrity": "sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ=="
|
||||
},
|
||||
"@szmarczak/http-timer": {
|
||||
"version": "4.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz",
|
||||
"integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==",
|
||||
"requires": {
|
||||
"defer-to-connect": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"@types/cacheable-request": {
|
||||
"version": "6.0.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz",
|
||||
"integrity": "sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==",
|
||||
"requires": {
|
||||
"@types/http-cache-semantics": "*",
|
||||
"@types/keyv": "*",
|
||||
"@types/node": "*",
|
||||
"@types/responselike": "*"
|
||||
}
|
||||
},
|
||||
"@types/cheerio": {
|
||||
"version": "0.22.22",
|
||||
"resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.22.tgz",
|
||||
@ -22,6 +46,11 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/http-cache-semantics": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz",
|
||||
"integrity": "sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A=="
|
||||
},
|
||||
"@types/jsonfile": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.0.0.tgz",
|
||||
@ -31,6 +60,14 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/keyv": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz",
|
||||
"integrity": "sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==",
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/node": {
|
||||
"version": "14.14.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.7.tgz",
|
||||
@ -45,6 +82,14 @@
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/responselike": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
|
||||
"integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/write": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/write/-/write-2.0.0.tgz",
|
||||
@ -129,6 +174,25 @@
|
||||
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
|
||||
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
|
||||
},
|
||||
"cacheable-lookup": {
|
||||
"version": "5.0.4",
|
||||
"resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
|
||||
"integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="
|
||||
},
|
||||
"cacheable-request": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz",
|
||||
"integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==",
|
||||
"requires": {
|
||||
"clone-response": "^1.0.2",
|
||||
"get-stream": "^5.1.0",
|
||||
"http-cache-semantics": "^4.0.0",
|
||||
"keyv": "^4.0.0",
|
||||
"lowercase-keys": "^2.0.0",
|
||||
"normalize-url": "^4.1.0",
|
||||
"responselike": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"cheerio": {
|
||||
"version": "1.0.0-rc.3",
|
||||
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
|
||||
@ -147,6 +211,14 @@
|
||||
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
|
||||
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
|
||||
},
|
||||
"clone-response": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",
|
||||
"integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=",
|
||||
"requires": {
|
||||
"mimic-response": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
@ -176,6 +248,26 @@
|
||||
"ms": "2.1.2"
|
||||
}
|
||||
},
|
||||
"decompress-response": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
|
||||
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
|
||||
"requires": {
|
||||
"mimic-response": "^3.1.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"mimic-response": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
|
||||
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"defer-to-connect": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz",
|
||||
"integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg=="
|
||||
},
|
||||
"devtools-protocol": {
|
||||
"version": "0.0.809251",
|
||||
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.809251.tgz",
|
||||
@ -302,6 +394,24 @@
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"got": {
|
||||
"version": "11.8.2",
|
||||
"resolved": "https://registry.npmjs.org/got/-/got-11.8.2.tgz",
|
||||
"integrity": "sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==",
|
||||
"requires": {
|
||||
"@sindresorhus/is": "^4.0.0",
|
||||
"@szmarczak/http-timer": "^4.0.5",
|
||||
"@types/cacheable-request": "^6.0.1",
|
||||
"@types/responselike": "^1.0.0",
|
||||
"cacheable-lookup": "^5.0.3",
|
||||
"cacheable-request": "^7.0.1",
|
||||
"decompress-response": "^6.0.0",
|
||||
"http2-wrapper": "^1.0.0-beta.5.2",
|
||||
"lowercase-keys": "^2.0.0",
|
||||
"p-cancelable": "^2.0.0",
|
||||
"responselike": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "4.2.4",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz",
|
||||
@ -320,6 +430,20 @@
|
||||
"readable-stream": "^3.1.1"
|
||||
}
|
||||
},
|
||||
"http-cache-semantics": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
|
||||
"integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
|
||||
},
|
||||
"http2-wrapper": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz",
|
||||
"integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==",
|
||||
"requires": {
|
||||
"quick-lru": "^5.1.1",
|
||||
"resolve-alpn": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"https-proxy-agent": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
|
||||
@ -348,6 +472,11 @@
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
},
|
||||
"json-buffer": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
|
||||
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
|
||||
},
|
||||
"jsonfile": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
|
||||
@ -357,6 +486,14 @@
|
||||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"keyv": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz",
|
||||
"integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==",
|
||||
"requires": {
|
||||
"json-buffer": "3.0.1"
|
||||
}
|
||||
},
|
||||
"locate-path": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
|
||||
@ -370,6 +507,16 @@
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
|
||||
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
|
||||
},
|
||||
"lowercase-keys": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
|
||||
"integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
|
||||
},
|
||||
"mimic-response": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
|
||||
"integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ=="
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
@ -393,6 +540,11 @@
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
},
|
||||
"normalize-url": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
|
||||
"integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ=="
|
||||
},
|
||||
"nth-check": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
|
||||
@ -409,6 +561,11 @@
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"p-cancelable": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.0.tgz",
|
||||
"integrity": "sha512-HAZyB3ZodPo+BDpb4/Iu7Jv4P6cSazBz9ZM0ChhEXp70scx834aWCEjQRwgt41UzzejUAPdbqqONfRWTPYrPAQ=="
|
||||
},
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
@ -499,6 +656,11 @@
|
||||
"ws": "^7.2.3"
|
||||
}
|
||||
},
|
||||
"quick-lru": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
|
||||
"integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "3.6.0",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
|
||||
@ -509,6 +671,19 @@
|
||||
"util-deprecate": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"resolve-alpn": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.0.0.tgz",
|
||||
"integrity": "sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA=="
|
||||
},
|
||||
"responselike": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
|
||||
"integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
|
||||
"requires": {
|
||||
"lowercase-keys": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.3",
|
||||
"fs-extra": "^9.0.1",
|
||||
"got": "^11.8.2",
|
||||
"jsonfile": "^6.1.0",
|
||||
"puppeteer": "^5.4.1",
|
||||
"tldts": "^5.6.71",
|
||||
|
||||
50
src/core.ts
50
src/core.ts
@ -3,6 +3,7 @@ import jsonfile from "jsonfile";
|
||||
import cheerio from "cheerio";
|
||||
import fs from "fs-extra";
|
||||
import { getDomain } from "tldts";
|
||||
import got from "got";
|
||||
|
||||
// Import website modules
|
||||
import {
|
||||
@ -26,7 +27,10 @@ import {
|
||||
ZooEnzo,
|
||||
PetsOnline,
|
||||
PetsPlace,
|
||||
ThePetEmpire
|
||||
ThePetEmpire,
|
||||
Conrad,
|
||||
HuisdierExpress,
|
||||
HondenBed
|
||||
} from "./modules/websites";
|
||||
|
||||
start();
|
||||
@ -35,20 +39,30 @@ start();
|
||||
let browser: puppeteer.Browser = null;
|
||||
let allProducts: Array<any> = [];
|
||||
let allBlogUrls: Array<any> = [];
|
||||
let urlsFileName = "";
|
||||
|
||||
async function start() {
|
||||
try {
|
||||
console.log(`---- ContentEggStockCrawler ----`);
|
||||
console.log(`---- ContentEggStockCrawler ----`);
|
||||
let params = process.argv.slice(2);
|
||||
|
||||
// Crawl Sitemap
|
||||
const sitemap = await got(params[0]);
|
||||
|
||||
const urls: Array<string> = [];
|
||||
const $ = cheerio.load(sitemap.body, { xmlMode: true })
|
||||
|
||||
$('loc').each(function() {
|
||||
const url = $(this).text()
|
||||
|
||||
if (!urls.includes(url)) {
|
||||
urls.push(url)
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Loaded ${urls.length} urls from sitemap! (${params[0]})`);
|
||||
|
||||
console.log(`Starting crawler browser..`);
|
||||
browser = await puppeteer.launch({ headless: true });
|
||||
|
||||
let params = process.argv.slice(2);
|
||||
urlsFileName = params[0].replace(/\.[^/.]+$/, "");
|
||||
|
||||
const urls = await jsonfile.readFile(`./${params[0]}`);
|
||||
console.log(`Loaded ${urls.length} urls from file! (${params[0]})`);
|
||||
|
||||
console.log(`Initialization done!`);
|
||||
console.log(`------------------------------------- \n`);
|
||||
@ -133,9 +147,9 @@ async function crawlBlogPage(url: string) {
|
||||
products[index].inStock = status[2];
|
||||
|
||||
if (products[index].inStock) {
|
||||
console.log(`✅ ${products[index].name} - ${products[index].domain}`);
|
||||
console.log(`IN STOCK! ${products[index].name} - ${products[index].domain}`);
|
||||
} else {
|
||||
console.log(`❌ ${products[index].name} - ${products[index].domain}`);
|
||||
console.log(`OUT OF STOCK! ${products[index].name} - ${products[index].domain}`);
|
||||
}
|
||||
|
||||
allProducts.push(products[index]);
|
||||
@ -208,6 +222,12 @@ async function crawlProductStock(url: string) {
|
||||
case "silvergear.eu":
|
||||
return [domain, page.url(), await Silvergear.check(html)];
|
||||
|
||||
case "conrad.com":
|
||||
return [domain, page.url(), await Conrad.check(html)];
|
||||
|
||||
case "conrad.nl":
|
||||
return [domain, page.url(), await Conrad.check(html)];
|
||||
|
||||
case "alsa-nature.nl":
|
||||
return [domain, page.url(), await AlsaNature.check(html)];
|
||||
|
||||
@ -240,6 +260,12 @@ async function crawlProductStock(url: string) {
|
||||
|
||||
case "thepetempire.com":
|
||||
return [domain, page.url(), await ThePetEmpire.check(html)];
|
||||
|
||||
case "huisdierexpress.nl":
|
||||
return [domain, page.url(), await HuisdierExpress.check(html)];
|
||||
|
||||
case "hondenbed.nl":
|
||||
return [domain, page.url(), await HondenBed.check(html)];
|
||||
|
||||
default:
|
||||
console.error(`-- ${domain} is not an available website module! Can't check stock!`);
|
||||
@ -254,7 +280,7 @@ async function crawlProductStock(url: string) {
|
||||
async function generateReport() {
|
||||
console.log("Generating report...");
|
||||
|
||||
let file = `${urlsFileName}-report.txt`;
|
||||
let file = `stock-report.txt`;
|
||||
let totalProducts = 0;
|
||||
let totalOutStock = 0;
|
||||
for (let product of allProducts) {
|
||||
|
||||
@ -166,7 +166,7 @@ export namespace BCC {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
if ($(".productoffer__availability .productoffer__deliverymsg .icon").length >= 1) {
|
||||
if ($(".productoffer__availability .productoffer__deliverymsg .icon").first().length >= 1) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
@ -183,7 +183,7 @@ export namespace Alternate {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
if ($(".productMainContainerRow .stockStatus.available_stock")) {
|
||||
if ($(".productMainContainerRow .stockStatus.available_stock").length >= 1) {
|
||||
if ($(".productMainContainerRow .stockStatus.available_stock").first().html().trim().toLocaleLowerCase() >= "direct leverbaar") {
|
||||
return true;
|
||||
} else {
|
||||
@ -217,6 +217,26 @@ export namespace Expert {
|
||||
}
|
||||
}
|
||||
|
||||
export namespace Conrad {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
if ($(".product-summary .availability__status")) {
|
||||
if ($(".product-summary .availability__status").first().html().trim().toLowerCase() == "available") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.error(`Error occured during stock check!`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export namespace Silvergear {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
@ -323,11 +343,15 @@ export namespace MacroVet {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
if ($('.product-essential meta[itemprop=availability]').attr("href") == "http://schema.org/InStock") {
|
||||
return true;
|
||||
}
|
||||
|
||||
if ($(".row-products-blog .row-products-blog2 .active link").attr("href") == "http://schema.org/InStock") {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.error(`Error occured during stock check!`);
|
||||
@ -419,4 +443,40 @@ export namespace ThePetEmpire {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export namespace HuisdierExpress {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
if ($("[itemprop=availability]").first().attr("content") == "in_stock") {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.error(`Error occured during stock check!`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export namespace HondenBed {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
if ($("[itemprop=availability]").first().attr("content") == "http://schema.org/InStock") {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.error(`Error occured during stock check!`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,128 +0,0 @@
|
||||
----------------------------------------
|
||||
Content Egg Product Stock Crawler Report
|
||||
Total Blog Urls: 16
|
||||
Total Products: 228
|
||||
Total Out of Stock: 56
|
||||
----------------------------------------
|
||||
|
||||
|
||||
https://sportschoolplus.nl/beste-eiwitpoeders/
|
||||
----------------------------------
|
||||
bol.com - Optimum nutrition GOLD STANDARD 100% WHEY PROTEIN - 908 gram - vanilla ice - https://www.bol.com/nl/p/optimum-nutrition-gold-standard-100-whey-protein-908-gram-vanilla-ice/9200000073275041/?Referrer=ADVNLPPcef22500002e88c20065bba51d000047814&utm_source=47814&utm_medium=Affiliates&utm_campaign=CPS&utm_content=txl
|
||||
|
||||
|
||||
https://sportschoolplus.nl/beste-weight-gainers/
|
||||
----------------------------------
|
||||
|
||||
|
||||
https://sportschoolplus.nl/cable-crossover-machines/
|
||||
----------------------------------
|
||||
fitnesskoerier.nl - Body-Solid GCCO150 Cable Crossover 2 x 75 kg - https://www.fitnesskoerier.nl/body-solid-gcco150-cable-crossover-2-x-75-kg.html?source=tradetracker
|
||||
fitnessapparaat.nl - Body-Solid GCCO150 Cable Crossover - https://www.fitnessapparaat.nl/artikel/6438/body-solid-gcco150-cable-crossover.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
fitnessapparaat.nl - Body-Solid GDCC200 Functional Training Center - Cable Crossover - https://www.fitnessapparaat.nl/artikel/6781/body-solid-gdcc200-functional-training-center-cable-crossover.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
fitnessapparaat.nl - Body-Solid GDCC250 Deluxe Cable Crossover - https://www.fitnessapparaat.nl/artikel/6564/body-solid-gdcc250-deluxe-cable-crossover.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
fitnesskoerier.nl - Best Fitness BFFT10 Functional Trainer - https://www.fitnesskoerier.nl/best-fitness-bfft10-functional-trainer.html?source=tradetracker
|
||||
|
||||
|
||||
https://sportschoolplus.nl/10-beste-spinningfietsen-vanaf-450-euro/
|
||||
----------------------------------
|
||||
fitnessapparaat.nl - Flow Fitness Racer DSB600i Speed Bike - Gratis trainingsschema - https://www.fitnessapparaat.nl/artikel/24743/flow-fitness-racer-dsb600i-speed-bike-gratis-trainingsschema.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
fitnessapparaat.nl - VirtuFit Indoor Cycle S1 Spinningfiets - Inclusief Spinning DVD - https://www.fitnessapparaat.nl/artikel/24939/virtufit-indoor-cycle-s1-spinningfiets-gratis-trainingsschema.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
fitnessapparaat.nl - VirtuFit Elite Indoor Cycle Pro Spinningfiets - Inclusief Spinning DVD - https://www.fitnessapparaat.nl/artikel/24942/virtufit-elite-indoor-cycle-pro-spinningfiets-gratis-trainingsschema.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
betersport.nl - Spinningbike - Spirit CB900 - https://www.betersport.nl/spinningbike-spirit-cb900.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Spinningbike - NordicTrack S10i - https://www.betersport.nl/spinningbike-nordictrack-s10i.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
|
||||
|
||||
https://sportschoolplus.nl/crosstrainers/
|
||||
----------------------------------
|
||||
sportschoolplus.nl - VirtuFit iConsole CTR 2.1 Ergometer Crosstrainer - Gratis trainingsschema - https://sportschoolplus.nl/product/14344-CTR2.1-73251
|
||||
betersport.nl - Crosstrainer - Focus Fitness Senator iPlus - https://www.betersport.nl/crosstrainer-focus-fitness-senator-iplus.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Gymost Endurance E15 - https://www.betersport.nl/crosstrainer-gymost-endurance-e15.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Gymost Endurance E15 - https://www.betersport.nl/crosstrainer-gymost-endurance-e15.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
bol.com - Tunturi Competence C20 Crosstrainer - https://www.bol.com/nl/p/tunturi-competence-c20-crosstrainer/9200000075670502/?Referrer=ADVNLPPcef22500002e88c20065bba51d000047814&utm_source=47814&utm_medium=Affiliates&utm_campaign=CPS&utm_content=txl
|
||||
betersport.nl - Crosstrainer - Schwinn 570E - https://www.betersport.nl/crosstrainer-schwinn-570e.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - NordicTrack New Commercial 12.9 - https://www.betersport.nl/crosstrainer-nordictrack-new-commercial-12-9.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Gymost E11 Legend - https://www.betersport.nl/crosstrainer-gymost-e11-legend.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Nautilus E628 - https://www.betersport.nl/crosstrainer-nautilus-e628.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Bowflex BXE226 Results Series - https://www.betersport.nl/crosstrainer-bowflex-bxe226-results-series.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Sole Fitness E95 - https://www.betersport.nl/crosstrainer-sole-fitness-e95.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Sole Fitness E95 - https://www.betersport.nl/crosstrainer-sole-fitness-e95.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Bowflex BXE326 Results Series - https://www.betersport.nl/crosstrainer-bowflex-bxe326-results-series.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Max Trainer - Bowflex M7i - https://www.betersport.nl/maxtrainer-bowflex-m7i.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Crosstrainer - Sole Fitness E95s - https://www.betersport.nl/crosstrainer-sole-fitness-e95s.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
|
||||
|
||||
https://sportschoolplus.nl/beste-halterbanken/
|
||||
----------------------------------
|
||||
fitnessapparaat.nl - VirtuFit Verstelbare Halterbank Compact - https://www.fitnessapparaat.nl/artikel/22734/virtufit-verstelbare-halterbank-compact.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
bol.com - Trainingsbank Buiktrainer Fitnessbank Halter Bank 401079 - https://www.bol.com/nl/p/trainingsbank-buiktrainer-fitnessbank-halter-bank-401079/9200000036361384/?Referrer=ADVNLPPcef22500002e88c20065bba51d000047814&utm_source=47814&utm_medium=Affiliates&utm_campaign=CPS&utm_content=txl
|
||||
bol.com - VirtuFit Verstelbare Halterbank Compact - https://www.bol.com/nl/p/virtufit-verstelbare-halterbank-compact/9200000116388343/?Referrer=ADVNLPPcef22500002e88c20065bba51d000047814&utm_source=47814&utm_medium=Affiliates&utm_campaign=CPS&utm_content=txl
|
||||
betersport.nl - Halterbank - Powerline POB144 - https://www.betersport.nl/halterbank-powerline-pob144.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
|
||||
|
||||
https://sportschoolplus.nl/dumbells-kopen-advies/
|
||||
----------------------------------
|
||||
bol.com - Tunturi Selector Dumbbell - Dumbbellset (2,5kg t/m 25kg) - https://www.bol.com/nl/p/tunturi-selector-dumbbell-dumbbellset/9200000076829318/?Referrer=ADVNLPPcef22500002e88c20065bba51d000047814&utm_source=47814&utm_medium=Affiliates&utm_campaign=CPS&utm_content=txl
|
||||
|
||||
|
||||
https://sportschoolplus.nl/handknijpers/
|
||||
----------------------------------
|
||||
|
||||
|
||||
https://sportschoolplus.nl/pull-up-bars/
|
||||
----------------------------------
|
||||
betersport.nl - Pull Up Bar - Body-Solid PUB34 - Doorway Chinning Bar - https://www.betersport.nl/body-solid-pub34-door-chinning-bar.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
|
||||
|
||||
https://sportschoolplus.nl/beste-roeitrainers/
|
||||
----------------------------------
|
||||
sportschoolplus.nl - Finnlo Aquon Water Flow Waterroeier - Roeitrainer - Gratis trainingsschema - https://sportschoolplus.nl/product/14029-F3706-16215
|
||||
betersport.nl - Roeitrainer - First Degree Mega PRO XL - https://www.betersport.nl/roeitrainer-first-degree-mega-pro-xl.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
|
||||
|
||||
https://sportschoolplus.nl/push-up-bars/
|
||||
----------------------------------
|
||||
bol.com - Aurae Push up Bar met Extra Grip - Professioneel Opdruksteunen - Fitness - Crossfit - Push up... - https://www.bol.com/nl/p/aurae-push-up-bar-met-extra-grip-professioneel-opdruksteunen-fitness-crossfit-push-up-stand-zwart/9300000002355870/?Referrer=ADVNLPPcef22500002e88c20065bba51d000047814&utm_source=47814&utm_medium=Affiliates&utm_campaign=CPS&utm_content=txl
|
||||
|
||||
|
||||
https://sportschoolplus.nl/loopbanden/
|
||||
----------------------------------
|
||||
betersport.nl - Loopband - Senz Sports M1000 - https://www.betersport.nl/loopband-senz-sports-m1000.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Loopband - Focus Fitness Jet 7 - https://www.betersport.nl/loopband-focus-fitness-jet-7.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Loopband - NordicTrack T7.0 - https://www.betersport.nl/loopband-nordictrack-t7-0.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Loopband - ProForm Pro 1000 - https://www.betersport.nl/loopband-proform-pro-1000.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Loopband - Sole Fitness F63 - https://www.betersport.nl/loopband-sole-f63.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Loopband - Schwinn 510T - https://www.betersport.nl/loopband-schwinn-510t.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
fitnessapparaat.nl - Bowflex BXT326 Result Series Loopband - Zwift Compatible - Gratis montage - https://www.fitnessapparaat.nl/artikel/21617/bowflex-bxt326-result-series-loopband-zwift-compatible-gratis-montage.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
betersport.nl - Loopband - NordicTrack New Commercial 2950 - https://www.betersport.nl/loopband-nordictrack-new-commercial-2950.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
|
||||
|
||||
https://sportschoolplus.nl/hometrainers/
|
||||
----------------------------------
|
||||
betersport.nl - Hometrainer - Sole Fitness B74 - https://www.betersport.nl/hometrainer-sole-fitness-b74.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Sole Fitness B74 - https://www.betersport.nl/hometrainer-sole-fitness-b74.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Nautilus U626 - https://www.betersport.nl/hometrainer-nautilus-u626.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Spirit XBU-55 - https://www.betersport.nl/hometrainer-spirit-xbu-55.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Sole Fitness LCB - https://www.betersport.nl/hometrainer-sole-fitness-lcb.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Sole Fitness LCB - https://www.betersport.nl/hometrainer-sole-fitness-lcb.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Spirit CU800 - https://www.betersport.nl/hometrainer-spirit-cu800.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
fitnessapparaat.nl - Nohrd Bike - Eiken - Gratis montage - https://www.fitnessapparaat.nl/artikel/24127/nohrd-bike-eiken-gratis-montage.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
betersport.nl - Hometrainer - Spirit CU800 - https://www.betersport.nl/hometrainer-spirit-cu800.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - ProForm 325 CSX+ - Recumbent Bike - https://www.betersport.nl/hometrainer-proform-325-csx-recumbent-bike.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Spirit CR800 - Recumbent Bike - https://www.betersport.nl/hometrainer-spirit-cr800-recumbent-bike.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
betersport.nl - Hometrainer - Sole Fitness LCB - https://www.betersport.nl/hometrainer-sole-fitness-lcb.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
|
||||
|
||||
https://sportschoolplus.nl/fitnesselastieken/
|
||||
----------------------------------
|
||||
bol.com - resistance band - weerstandsband- fitness elastiek- Elastische Weerstandsband 35 kg | Fitness... - https://www.bol.com/nl/p/kaytan-weerstandsband-fitness-elastiek-35-kg-rood/9200000132733796/?Referrer=ADVNLPPcef22500002e88c20065bba51d000047814&utm_source=47814&utm_medium=Affiliates&utm_campaign=CPS&utm_content=txl
|
||||
|
||||
|
||||
https://sportschoolplus.nl/foamrollers/
|
||||
----------------------------------
|
||||
|
||||
https://sportschoolplus.nl/de-beste-krachtstations-voor-thuis/
|
||||
----------------------------------
|
||||
betersport.nl - Home Gym - Powertec Levergym WB-LS Yellow - https://www.betersport.nl/home-gym-powertec-levergym-wb-ls.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
fitnessapparaat.nl - Finnlo Maximum Inspire - M3 Multi-gym - https://www.fitnessapparaat.nl/artikel/19807/finnlo-maximum-inspire-m3-multi-gym.html?utm_source=TradeTracker&utm_medium=Affiliate&utm_campaign=Sportschoolplus
|
||||
betersport.nl - Home Gym - Body-Solid G9S - https://www.betersport.nl/home-gym-body-solid-g9s.html?utm_source=tradetracker&utm_medium=affiliate&utm_term=179951&utm_campaign=betersport
|
||||
@ -1,18 +0,0 @@
|
||||
[
|
||||
"https://sportschoolplus.nl/beste-eiwitpoeders/",
|
||||
"https://sportschoolplus.nl/beste-weight-gainers/",
|
||||
"https://sportschoolplus.nl/cable-crossover-machines/",
|
||||
"https://sportschoolplus.nl/10-beste-spinningfietsen-vanaf-450-euro/",
|
||||
"https://sportschoolplus.nl/crosstrainers/",
|
||||
"https://sportschoolplus.nl/beste-halterbanken/",
|
||||
"https://sportschoolplus.nl/dumbells-kopen-advies/",
|
||||
"https://sportschoolplus.nl/handknijpers/",
|
||||
"https://sportschoolplus.nl/pull-up-bars/",
|
||||
"https://sportschoolplus.nl/beste-roeitrainers/",
|
||||
"https://sportschoolplus.nl/push-up-bars/",
|
||||
"https://sportschoolplus.nl/loopbanden/",
|
||||
"https://sportschoolplus.nl/hometrainers/",
|
||||
"https://sportschoolplus.nl/fitnesselastieken/",
|
||||
"https://sportschoolplus.nl/foamrollers/",
|
||||
"https://sportschoolplus.nl/de-beste-krachtstations-voor-thuis/"
|
||||
]
|
||||
Loading…
x
Reference in New Issue
Block a user