basic crawling system
This commit is contained in:
parent
09d814102e
commit
4fb8a19661
570
package-lock.json
generated
Normal file
570
package-lock.json
generated
Normal file
@ -0,0 +1,570 @@
|
||||
{
|
||||
"name": "contenteggstockcrawler",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"@types/cheerio": {
|
||||
"version": "0.22.22",
|
||||
"resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.22.tgz",
|
||||
"integrity": "sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/jsonfile": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.0.0.tgz",
|
||||
"integrity": "sha512-mUHbRieyluPtL3c466K7oUGua1lAVlz45PV4U3bHs5CXdBlDIeXJI5xQXa6IZYnrgmcJzJp/CiTZB4zfShAi6w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/node": {
|
||||
"version": "14.14.7",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.7.tgz",
|
||||
"integrity": "sha512-Zw1vhUSQZYw+7u5dAwNbIA9TuTotpzY/OF7sJM9FqPOF3SPjKnxrjoTktXDZgUjybf4cWVBP7O8wvKdSaGHweg=="
|
||||
},
|
||||
"@types/puppeteer": {
|
||||
"version": "5.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-5.4.0.tgz",
|
||||
"integrity": "sha512-zTYDLjnHjgzokrwKt7N0rgn7oZPYo1J0m8Ghu+gXqzLCEn8RWbELa2uprE2UFJ0jU/Sk0x9jXXdOH/5QQLFHhQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/yauzl": {
|
||||
"version": "2.9.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz",
|
||||
"integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==",
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"agent-base": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz",
|
||||
"integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g=="
|
||||
},
|
||||
"balanced-match": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
|
||||
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
|
||||
},
|
||||
"base64-js": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
|
||||
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
|
||||
},
|
||||
"bl": {
|
||||
"version": "4.0.3",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
|
||||
"integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
|
||||
"requires": {
|
||||
"buffer": "^5.5.0",
|
||||
"inherits": "^2.0.4",
|
||||
"readable-stream": "^3.4.0"
|
||||
}
|
||||
},
|
||||
"boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
|
||||
},
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
}
|
||||
},
|
||||
"buffer": {
|
||||
"version": "5.7.1",
|
||||
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
|
||||
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
|
||||
"requires": {
|
||||
"base64-js": "^1.3.1",
|
||||
"ieee754": "^1.1.13"
|
||||
}
|
||||
},
|
||||
"buffer-crc32": {
|
||||
"version": "0.2.13",
|
||||
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
|
||||
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
|
||||
},
|
||||
"cheerio": {
|
||||
"version": "1.0.0-rc.3",
|
||||
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
|
||||
"integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
|
||||
"requires": {
|
||||
"css-select": "~1.2.0",
|
||||
"dom-serializer": "~0.1.1",
|
||||
"entities": "~1.1.1",
|
||||
"htmlparser2": "^3.9.1",
|
||||
"lodash": "^4.15.0",
|
||||
"parse5": "^3.0.1"
|
||||
}
|
||||
},
|
||||
"chownr": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
|
||||
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
|
||||
},
|
||||
"concat-map": {
|
||||
"version": "0.0.1",
|
||||
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
|
||||
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
|
||||
},
|
||||
"css-select": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
|
||||
"integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
|
||||
"requires": {
|
||||
"boolbase": "~1.0.0",
|
||||
"css-what": "2.1",
|
||||
"domutils": "1.5.1",
|
||||
"nth-check": "~1.0.1"
|
||||
}
|
||||
},
|
||||
"css-what": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
|
||||
"integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
|
||||
},
|
||||
"debug": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
|
||||
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
|
||||
"requires": {
|
||||
"ms": "2.1.2"
|
||||
}
|
||||
},
|
||||
"devtools-protocol": {
|
||||
"version": "0.0.809251",
|
||||
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.809251.tgz",
|
||||
"integrity": "sha512-pf+2OY6ghMDPjKkzSWxHMq+McD+9Ojmq5XVRYpv/kPd9sTMQxzEt21592a31API8qRjro0iYYOc3ag46qF/1FA=="
|
||||
},
|
||||
"dom-serializer": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
|
||||
"integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
|
||||
"requires": {
|
||||
"domelementtype": "^1.3.0",
|
||||
"entities": "^1.1.1"
|
||||
}
|
||||
},
|
||||
"domelementtype": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
|
||||
"integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
|
||||
},
|
||||
"domhandler": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
|
||||
"integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
|
||||
"requires": {
|
||||
"domelementtype": "1"
|
||||
}
|
||||
},
|
||||
"domutils": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
|
||||
"integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
|
||||
"requires": {
|
||||
"dom-serializer": "0",
|
||||
"domelementtype": "1"
|
||||
}
|
||||
},
|
||||
"end-of-stream": {
|
||||
"version": "1.4.4",
|
||||
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
|
||||
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
|
||||
"requires": {
|
||||
"once": "^1.4.0"
|
||||
}
|
||||
},
|
||||
"entities": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
|
||||
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
|
||||
},
|
||||
"extract-zip": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
|
||||
"integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
|
||||
"requires": {
|
||||
"@types/yauzl": "^2.9.1",
|
||||
"debug": "^4.1.1",
|
||||
"get-stream": "^5.1.0",
|
||||
"yauzl": "^2.10.0"
|
||||
}
|
||||
},
|
||||
"fd-slicer": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
|
||||
"integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
|
||||
"requires": {
|
||||
"pend": "~1.2.0"
|
||||
}
|
||||
},
|
||||
"find-up": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
|
||||
"integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
|
||||
"requires": {
|
||||
"locate-path": "^5.0.0",
|
||||
"path-exists": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"fs-constants": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
|
||||
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
|
||||
},
|
||||
"fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
|
||||
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
|
||||
},
|
||||
"get-stream": {
|
||||
"version": "5.2.0",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
|
||||
"integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
|
||||
"requires": {
|
||||
"pump": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"glob": {
|
||||
"version": "7.1.6",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
|
||||
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
|
||||
"requires": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "4.2.4",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz",
|
||||
"integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==",
|
||||
"optional": true
|
||||
},
|
||||
"htmlparser2": {
|
||||
"version": "3.10.1",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
|
||||
"integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
|
||||
"requires": {
|
||||
"domelementtype": "^1.3.1",
|
||||
"domhandler": "^2.3.0",
|
||||
"domutils": "^1.5.1",
|
||||
"entities": "^1.1.1",
|
||||
"inherits": "^2.0.1",
|
||||
"readable-stream": "^3.1.1"
|
||||
}
|
||||
},
|
||||
"https-proxy-agent": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
|
||||
"integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==",
|
||||
"requires": {
|
||||
"agent-base": "5",
|
||||
"debug": "4"
|
||||
}
|
||||
},
|
||||
"ieee754": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
|
||||
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
|
||||
},
|
||||
"inflight": {
|
||||
"version": "1.0.6",
|
||||
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
|
||||
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
|
||||
"requires": {
|
||||
"once": "^1.3.0",
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
},
|
||||
"jsonfile": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz",
|
||||
"integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==",
|
||||
"requires": {
|
||||
"graceful-fs": "^4.1.6",
|
||||
"universalify": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"locate-path": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
|
||||
"integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
|
||||
"requires": {
|
||||
"p-locate": "^4.1.0"
|
||||
}
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.20",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
|
||||
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
|
||||
},
|
||||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
},
|
||||
"mkdirp-classic": {
|
||||
"version": "0.5.3",
|
||||
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
|
||||
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
|
||||
},
|
||||
"ms": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
|
||||
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
},
|
||||
"nth-check": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
|
||||
"integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
|
||||
"requires": {
|
||||
"boolbase": "~1.0.0"
|
||||
}
|
||||
},
|
||||
"once": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
|
||||
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
|
||||
"requires": {
|
||||
"wrappy": "1"
|
||||
}
|
||||
},
|
||||
"p-limit": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
|
||||
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
|
||||
"requires": {
|
||||
"p-try": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"p-locate": {
|
||||
"version": "4.1.0",
|
||||
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
|
||||
"integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
|
||||
"requires": {
|
||||
"p-limit": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"p-try": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
|
||||
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
|
||||
},
|
||||
"parse5": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
|
||||
"integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"path-exists": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
|
||||
"integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
|
||||
},
|
||||
"path-is-absolute": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
|
||||
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
|
||||
},
|
||||
"pend": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
|
||||
"integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
|
||||
},
|
||||
"pkg-dir": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
|
||||
"integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
|
||||
"requires": {
|
||||
"find-up": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"progress": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
|
||||
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
|
||||
},
|
||||
"proxy-from-env": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
|
||||
},
|
||||
"pump": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
|
||||
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
|
||||
"requires": {
|
||||
"end-of-stream": "^1.1.0",
|
||||
"once": "^1.3.1"
|
||||
}
|
||||
},
|
||||
"puppeteer": {
|
||||
"version": "5.4.1",
|
||||
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.4.1.tgz",
|
||||
"integrity": "sha512-8u6r9tFm3gtMylU4uCry1W/CeAA8uczKMONvGvivkTsGqKA7iB7DWO2CBFYlB9GY6/IEoq9vkI5slJWzUBkwNw==",
|
||||
"requires": {
|
||||
"debug": "^4.1.0",
|
||||
"devtools-protocol": "0.0.809251",
|
||||
"extract-zip": "^2.0.0",
|
||||
"https-proxy-agent": "^4.0.0",
|
||||
"node-fetch": "^2.6.1",
|
||||
"pkg-dir": "^4.2.0",
|
||||
"progress": "^2.0.1",
|
||||
"proxy-from-env": "^1.0.0",
|
||||
"rimraf": "^3.0.2",
|
||||
"tar-fs": "^2.0.0",
|
||||
"unbzip2-stream": "^1.3.3",
|
||||
"ws": "^7.2.3"
|
||||
}
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "3.6.0",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
|
||||
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
|
||||
"requires": {
|
||||
"inherits": "^2.0.3",
|
||||
"string_decoder": "^1.1.1",
|
||||
"util-deprecate": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
|
||||
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
|
||||
"requires": {
|
||||
"glob": "^7.1.3"
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
|
||||
"requires": {
|
||||
"safe-buffer": "~5.2.0"
|
||||
}
|
||||
},
|
||||
"tar-fs": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
|
||||
"integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
|
||||
"requires": {
|
||||
"chownr": "^1.1.1",
|
||||
"mkdirp-classic": "^0.5.2",
|
||||
"pump": "^3.0.0",
|
||||
"tar-stream": "^2.1.4"
|
||||
}
|
||||
},
|
||||
"tar-stream": {
|
||||
"version": "2.1.4",
|
||||
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz",
|
||||
"integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==",
|
||||
"requires": {
|
||||
"bl": "^4.0.3",
|
||||
"end-of-stream": "^1.4.1",
|
||||
"fs-constants": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
"readable-stream": "^3.1.1"
|
||||
}
|
||||
},
|
||||
"through": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
|
||||
},
|
||||
"tldts": {
|
||||
"version": "5.6.71",
|
||||
"resolved": "https://registry.npmjs.org/tldts/-/tldts-5.6.71.tgz",
|
||||
"integrity": "sha512-4sC+aVjRi99cJWJIMWcuM4fX9mlw0NKpLhOF/gZwKYtbSGBXjLAIUJ43wJkMFn3eLUy938hyKmdvAvke7OWEyw==",
|
||||
"requires": {
|
||||
"tldts-core": "^5.6.71"
|
||||
}
|
||||
},
|
||||
"tldts-core": {
|
||||
"version": "5.6.71",
|
||||
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.6.71.tgz",
|
||||
"integrity": "sha512-tIrUPFN247p7bdubl9oaJ7ctJ5tMYo1UFUl5Su3ZC9s+l+feYjy2YW8basgTibTHc5S0wtvRdheE0hkC1b21Xw=="
|
||||
},
|
||||
"typescript": {
|
||||
"version": "4.0.5",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.0.5.tgz",
|
||||
"integrity": "sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ=="
|
||||
},
|
||||
"unbzip2-stream": {
|
||||
"version": "1.4.3",
|
||||
"resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
|
||||
"integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
|
||||
"requires": {
|
||||
"buffer": "^5.2.1",
|
||||
"through": "^2.3.8"
|
||||
}
|
||||
},
|
||||
"universalify": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz",
|
||||
"integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ=="
|
||||
},
|
||||
"util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
|
||||
},
|
||||
"wrappy": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
|
||||
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
|
||||
},
|
||||
"ws": {
|
||||
"version": "7.4.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-7.4.0.tgz",
|
||||
"integrity": "sha512-kyFwXuV/5ymf+IXhS6f0+eAFvydbaBW3zjpT6hUdAh/hbVjTIB5EHBGi0bPoCLSK2wcuz3BrEkB9LrYv1Nm4NQ=="
|
||||
},
|
||||
"yauzl": {
|
||||
"version": "2.10.0",
|
||||
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
|
||||
"integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
|
||||
"requires": {
|
||||
"buffer-crc32": "~0.2.3",
|
||||
"fd-slicer": "~1.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
27
package.json
Normal file
27
package.json
Normal file
@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "contenteggstockcrawler",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "./dist/core.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://git.aterve.com/Cryogenix_/ContentEggStockCrawler.git"
|
||||
},
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.3",
|
||||
"jsonfile": "^6.1.0",
|
||||
"puppeteer": "^5.4.1",
|
||||
"tldts": "^5.6.71",
|
||||
"typescript": "^4.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/cheerio": "^0.22.22",
|
||||
"@types/jsonfile": "^6.0.0",
|
||||
"@types/puppeteer": "^5.4.0"
|
||||
}
|
||||
}
|
||||
129
src/core.ts
Normal file
129
src/core.ts
Normal file
@ -0,0 +1,129 @@
|
||||
import puppeteer from "puppeteer";
|
||||
import jsonfile from "jsonfile";
|
||||
import cheerio from "cheerio";
|
||||
import { getDomain } from "tldts";
|
||||
|
||||
// Import website modules
|
||||
import {
|
||||
FitnessKoerier,
|
||||
FitnessApparaat
|
||||
} from "./modules/websites";
|
||||
|
||||
start();
|
||||
|
||||
// Globals
|
||||
let browser: puppeteer.Browser = null;
|
||||
|
||||
async function start() {
|
||||
try {
|
||||
console.log(`---- ContentEggStockCrawler ----`);
|
||||
|
||||
console.log(`Starting crawler browser..`);
|
||||
browser = await puppeteer.launch({ headless: true });
|
||||
|
||||
const urls = await jsonfile.readFile("./urls-check.json");
|
||||
console.log(`Loaded ${urls.length} urls from file! (urls-check.json)`);
|
||||
|
||||
console.log(`Initialization done!`);
|
||||
console.log(`------------------------------------- \n`);
|
||||
|
||||
for (let url of urls) {
|
||||
await crawlBlogPage(url);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error(`An Error Occured!`, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function crawlBlogPage(url: string) {
|
||||
try {
|
||||
console.log(`Crawling blog page: ${url}`);
|
||||
|
||||
// Open new page and goto url
|
||||
const page = await browser.newPage();
|
||||
await page.goto(url, { waitUntil: 'networkidle2' });
|
||||
|
||||
// Get html from page
|
||||
const html = await page.evaluate(() => document.body.innerHTML);
|
||||
|
||||
// Close page
|
||||
await page.close();
|
||||
|
||||
// Load html in cheerio object
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
let products: Array<any> = [];
|
||||
|
||||
console.log(`Detected ${$(".row-products").length} content egg products!`);
|
||||
|
||||
$(".row-products .cegg-list-logo-title a").each((index, element) => {
|
||||
let url = $(element).attr("href");
|
||||
let name = $(element).html().trim();
|
||||
|
||||
products.push({
|
||||
name: name,
|
||||
blogUrl: url,
|
||||
url: "",
|
||||
domain: "",
|
||||
inStock: false,
|
||||
});
|
||||
});
|
||||
|
||||
console.log("Checking product stocks...");
|
||||
|
||||
for (let index in products) {
|
||||
let status = await crawlProductStock(products[index].blogUrl);
|
||||
|
||||
products[index].domain = status[0];
|
||||
products[index].url = status[1];
|
||||
products[index].inStock = status[2];
|
||||
|
||||
if (products[index].inStock) {
|
||||
console.log(`✅ ${products[index].name} - ${products[index].domain}`);
|
||||
} else {
|
||||
console.log(`❌ ${products[index].name} - ${products[index].domain}`);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.log("Error while trying to crawl page! Skipping...");
|
||||
}
|
||||
}
|
||||
|
||||
async function crawlProductStock(url: string) {
|
||||
try {
|
||||
// Open new page and goto url
|
||||
const page = await browser.newPage();
|
||||
await page.goto(url, { waitUntil: 'networkidle2' });
|
||||
|
||||
// Get html from page
|
||||
const html = await page.evaluate(() => document.body.innerHTML);
|
||||
|
||||
// Load html in cheerio object
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
// Get domain from url
|
||||
const domain = getDomain(page.url());
|
||||
|
||||
// Close page
|
||||
await page.close();
|
||||
|
||||
switch (domain) {
|
||||
case "fitnesskoerier.nl":
|
||||
return [domain, page.url(), await FitnessKoerier.check(html)];
|
||||
|
||||
case "fitnessapparaat.nl":
|
||||
return [domain, page.url(), await FitnessApparaat.check(html)];
|
||||
|
||||
default:
|
||||
console.error(`-- ${domain} is not an available website module! Can't check stock!`);
|
||||
return ["NOT_MODULE", false];
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
console.log("Error while trying to crawl page! Skipping...");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
46
src/modules/websites.ts
Normal file
46
src/modules/websites.ts
Normal file
@ -0,0 +1,46 @@
|
||||
import cheerio from "cheerio";
|
||||
import { isTypeReferenceNode } from "typescript";
|
||||
|
||||
export namespace Template {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
console.error(`Error occured during stock check!`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export namespace FitnessKoerier {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
if ($('.product-list .stock-msg').find("[itemprop=availability]").attr("content") == "out_of_stock") {
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error occured during stock check!`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export namespace FitnessApparaat {
|
||||
export async function check(html: string) {
|
||||
try {
|
||||
const $ = cheerio.load(html);
|
||||
console.log($('[itemprop="offers"').find(".stock-red").length);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
console.error(`Error occured during stock check!`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
16
tsconfig.json
Normal file
16
tsconfig.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"module": "commonjs",
|
||||
"esModuleInterop": true,
|
||||
"target": "es6",
|
||||
"noImplicitAny": true,
|
||||
"moduleResolution": "node",
|
||||
"sourceMap": true,
|
||||
"outDir": "dist",
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"*": ["node_modules/*", "src/types/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src/**/*"]
|
||||
}
|
||||
3
urls-check.json
Normal file
3
urls-check.json
Normal file
@ -0,0 +1,3 @@
|
||||
[
|
||||
"https://sportschoolplus.nl/cable-crossover-machines/"
|
||||
]
|
||||
Loading…
x
Reference in New Issue
Block a user