version 2.0.0 features added

Author: kayslay

UPDATE.md

@@ -0,0 +1,16 @@
+# V 2.0.0
+
+- ## added dynamicSchemas config option
+    web-crawljs now has dynamicSchemas that allow you change the structure of nextSelector, nextSelectBy,
+     fetchSelector and fetchSelectBy if the url matches a url yu specify.
+    
+- ## changed loop and loopFn to depth and depthFn
+     The loop and loopFn have been renamed to depth and depthFn.      
+      
+- ## fixed issue with finalFn
+    The issue with finalFn in the formal version is now fixed.
+      
+- ## added formatUrl config option
+    The function format the next url to crawl. It takes a url and returns a new url. this url 
+    can be a string or an Object that is supported by the request module for making requests.
+    

module/crawl.js

@@ -8,12 +8,13 @@ function createCrawler(config = {}) {
 
     let urls, finalFn, depthFn, depth;
 
+    //
     function defaultLoopFn(data) {
-        console.log("end of each loop")
+        console.log("---depth---")
     }
 
     function defaultFinalFn() {
-        console.log('end...')
+        console.log('---final---')
     }
 
 
@@ -28,26 +29,30 @@ function createCrawler(config = {}) {
         nextLinks = nextLinks.concat(urls);
     })(config);
 
+    /**
+     * @description
+     */
     function crawl() {
-        try {
+
             crawlUrls(nextLinks, config)
                 .then(scrapedData => {
                     depthFn(scrapedData.fetchedData);
                     gen.next(scrapedData.nextLinks);
                 })
                 .catch(err => {
-                    console.error(err)
-                })
-        }catch (err){
-            console.log(err)
-        }
+                    gen.next({err})
+                });
+
     }
 
 
     function* crawlGen() {
         for (let i = 0; i < depth; i++) {
             nextLinks = yield crawl();
-            // console.log(nextLinks)
+            if(nextLinks.err){
+                console.error(nextLinks.err);
+                break;
+            }
         }
         finalFn()
     }

module/crawlUrls.js

@@ -52,15 +52,18 @@ function crawlUrl(urls, resolve) {
     function req(url) {
 
         request(url, function (err, response, body) {
-            if (err) return console.log(err.message);
             visitedUrls--;
-            getDomContents = dom(body).getDomContents; //
-            scrapedData.push(fetchFromPage(url));
-            let newLink = _.uniq(util.sortDataToArray([selectNextCrawlContent(url)])).map(url => {
-                "use strict";
-                return formatUrl(url)
-            });
-            initialLink = initialLink.concat(newLink);
+            if (err) {
+                console.error(err.message);
+            } else {
+                getDomContents = dom(body).getDomContents; //
+                scrapedData.push(fetchFromPage(url));
+                let newLink = _.uniq(util.sortDataToArray([selectNextCrawlContent(url)])).map(url => {
+                    "use strict";
+                    return formatUrl(url)
+                });
+                initialLink = initialLink.concat(newLink);
+            }
 
             if (visitedUrls == 0) {
                 resolve({fetchedData: scrapedData, nextLinks: initialLink})

module/dom.js

@@ -8,7 +8,14 @@ module.exports = function (content) {
     //errors
     const KEY_ERROR = "the keys don't match. Make sure keys in arg1 matches keys in arg2";
 
-
+    /**
+     * @description returns back the Dom content from the page's body
+     * @param selector
+     * @param selectBy
+     * @param callback
+     * @param url
+     * @return {*}
+     */
     function getDomContents(selector, selectBy, callback, url) {
 
         let $ = cheerio.load(content);
@@ -57,7 +64,7 @@ module.exports = function (content) {
 
 
     /**
-     *
+     * @description extracts the data by the name given to the seector Object keys.
      * @param selector
      * @param selectBy
      * @param $

module/util.js

@@ -62,7 +62,6 @@ function sortDataToArray(data) {
     }, [])
 }
 
-//Todo: add what will reformat the urls, the selectBy e.t.c.
 /**
  * @description returns a url; the url can either be a string or an Object supported by request package
  * @param {String} url

test/crawl_test.js

@@ -1,14 +1,18 @@
 let expect = require('chai').expect;
 let Crawler = require('../index')({
-    fetchSelector: {title: "title", body: "div#mw-content-text"},
-    fetchSelectBy: {title: 'text', body: "text"},
+    fetchSelector: {title: "title"},
+    fetchSelectBy: {title: 'text'},
     nextSelector: {links: 'a[href^="/"]'},
     nextSelectBy: {links: ['attr', 'href']},
     fetchFn: (err, data, url) => {
         if (err) console.error(err.message);
-        if (url == 'http://localhost/dashboard/') console.log('saving somewhere', data);
-        // console.log(data.title[0],url)
-        // require('fs').writeFile('./data.txt',data.body[0],(err,data)=>console.log('data written to data.txt'))
+        if (/https:\/\/en.wiki/.test(url)) {
+            let json = JSON.stringify(data);
+            require('fs').writeFile('./data.json', json, (err, data) => console.log('data written to data.txt'))
+        } else {
+            return console.log(data);
+        }
+
     },
     formatUrl: function (url) {
         if (url == 'http://localhost/dashboard/faq.html') {
@@ -21,14 +25,17 @@ let Crawler = require('../index')({
         // console.log(data,url)
     },
     dynamicSchemas: {
-        fetchSelector: [{url: /http:\/\/localhost\//, schema: {title: "title"}}],
-        fetchSelectBy: [{url: /http:\/\/localhost\//, schema: {title: "text"}}],
+        //when the url matches https://en.wikipedia.org it uses this schema to format it
+        fetchSelector: [{url: /https:\/\/en.wikipedia.org/, schema: {title: "title", body: "div#mw-content-text"}}],
+        fetchSelectBy: [{url: /https:\/\/en.wikipedia.org/, schema: {title: "text",body: "text"}}],
+        nextSelector: [{url: /https:\/\/en.wikipedia/, schema: {link: ""}}],
+        nextSelectBy: [{url: /https:\/\/en.wikipedia/, schema: {link: ""}}],
     },
     depthFn: function (data) {
         // console.log(data)
     },
     depth: 2,
-    urls: ['http://localhost/dashboard/']
+    urls: ['http://localhost/dashboard/' /*'https://en.wikipedia.org/wiki/Web_crawler'*/]
 });