const {
  Scraper,
  Root,
  DownloadContent,
  OpenLinks,
  CollectContent,
} = require("nodejs-web-scraper");
const fs = require("fs");

(async () => {
  const config = {
    baseSiteUrl: `https://www.some-news-site.com/`,
    startUrl: `https://www.some-news-site.com/`,
    filePath: "./images/",
    concurrency: 10, //Maximum concurrent jobs. More than 10 is not recommended.Default is 3.
    maxRetries: 3, //The scraper will try to repeat a failed request few times(excluding 404). Default is 5.
    logPath: "./logs/", //Highly recommended: Creates a friendly JSON for each operation object, with all the relevant data.
  };

  const scraper = new Scraper(config); //Create a new Scraper instance, and pass config to it.

  //Now we create the "operations" we need:

  const root = new Root(); //The root object fetches the startUrl, and starts the process.

  //Any valid cheerio selector can be passed. For further reference: https://cheerio.js.org/
  const category = new OpenLinks(".category", { name: "category" }); //Opens each category page.

  const article = new OpenLinks("article a", { name: "article" }); //Opens each article page.

  const image = new DownloadContent("img", { name: "image" }); //Downloads images.

  const title = new CollectContent("h1", { name: "title" }); //"Collects" the text from each H1 element.

  const story = new CollectContent("section.content", { name: "story" }); //"Collects" the the article body.

  root.addOperation(category); //Then we create a scraping "tree":
  category.addOperation(article);
  article.addOperation(image);
  article.addOperation(title);
  article.addOperation(story);

  await scraper.scrape(root);

  const articles = article.getData(); //Will return an array of all article objects(from all categories), each
  //containing its "children"(titles,stories and the downloaded image urls)

  //If you just want to get the stories, do the same with the "story" variable:
  const stories = story.getData();

  fs.writeFile("./articles.json", JSON.stringify(articles), () => {}); //Will produce a formatted JSON containing all article pages and their selected data.

  fs.writeFile("./stories.json", JSON.stringify(stories), () => {});
})();

const {
  Scraper,
  Root,
  OpenLinks,
  CollectContent,
  DownloadContent,
} = require("nodejs-web-scraper");
const fs = require("fs");

(async () => {
  const pages = []; //All ad pages.

  //pageObject will be formatted as {title,phone,images}, becuase these are the names we chose for the scraping operations below.
  //Note that each key is an array, because there might be multiple elements fitting the querySelector.
  //This hook is called after every page finished scraping.
  //It will also get an address argument.
  const getPageObject = (pageObject, address) => {
    pages.push(pageObject);
  };

  const config = {
    baseSiteUrl: `https://www.profesia.sk`,
    startUrl: `https://www.profesia.sk/praca/`,
    filePath: "./images/",
    logPath: "./logs/",
  };

  const scraper = new Scraper(config);

  const root = new Root(); //Open pages 1-10. You need to supply the querystring that the site uses(more details in the API docs).

  const jobAds = new OpenLinks(".list-row h2 a", {
    name: "Ad page",
    getPageObject,
  }); //Opens every job ad, and calls the getPageObject, passing the formatted dictionary.

  const phones = new CollectContent(".details-desc a.tel", { name: "phone" }); //Important to choose a name, for the getPageObject to produce the expected results.

  const titles = new CollectContent("h1", { name: "title" });

  root.addOperation(jobAds);
  jobAds.addOperation(titles);
  jobAds.addOperation(phones);

  await scraper.scrape(root);

  fs.writeFile("./pages.json", JSON.stringify(pages), () => {}); //Produces a formatted JSON with all job ads.
})();

const { Scraper, Root, DownloadContent } = require("nodejs-web-scraper");

(async () => {
  const config = {
    baseSiteUrl: `https://spectator.sme.sk`, //Important to provide the base url, which is the same as the starting url, in this example.
    startUrl: `https://spectator.sme.sk/`,
    filePath: "./images/",
    cloneFiles: true, //Will create a new image file with an appended name, if the name already exists. Default is false.
  };

  const scraper = new Scraper(config);

  const root = new Root(); //Root corresponds to the config.startUrl. This object starts the entire process

  const images = new DownloadContent("img"); //Create an operation that downloads all image tags in a given page(any Cheerio selector can be passed).

  root.addOperation(images); //We want to download the images from the root page, we need to Pass the "images" operation to the root.

  await scraper.scrape(root); //Pass the Root to the Scraper.scrape() and you're done.
})();

const { Scraper, Root, CollectContent } = require("nodejs-web-scraper");

(async () => {
  const config = {
    baseSiteUrl: `https://spectator.sme.sk`,
    startUrl: `https://spectator.sme.sk/`,
  };

  function getElementContent(element) {
    // Do something...
  }

  const scraper = new Scraper(config);

  const root = new Root();

  const title = new CollectContent(".first_class, .second_class", {
    getElementContent,
  }); //Any of these will fit.

  root.addOperation(title);

  await scraper.scrape(root);
})();

const {
  Scraper,
  Root,
  OpenLinks,
  CollectContent,
  DownloadContent,
} = require("nodejs-web-scraper");
const fs = require("fs");

(async () => {
  const pages = []; //All ad pages.

  //pageObject will be formatted as {title,phone,images}, becuase these are the names we chose for the scraping operations below.
  const getPageObject = (pageObject, address) => {
    pages.push(pageObject);
  };

  const config = {
    baseSiteUrl: `https://www.profesia.sk`,
    startUrl: `https://www.profesia.sk/praca/`,
    filePath: "./images/",
    logPath: "./logs/",
  };

  const scraper = new Scraper(config);

  const root = new Root({
    pagination: { queryString: "page_num", begin: 1, end: 10 },
  }); //Open pages 1-10.
  // YOU NEED TO SUPPLY THE QUERYSTRING that the site uses(more details in the API docs). "page_num" is just the string used on this example site.

  const jobAds = new OpenLinks(".list-row h2 a", {
    name: "Ad page",
    getPageObject,
  }); //Opens every job ad, and calls the getPageObject, passing the formatted object.

  const phones = new CollectContent(".details-desc a.tel", { name: "phone" }); //Important to choose a name, for the getPageObject to produce the expected results.

  const images = new DownloadContent("img", { name: "images" });

  const titles = new CollectContent("h1", { name: "title" });

  root.addOperation(jobAds);
  jobAds.addOperation(titles);
  jobAds.addOperation(phones);
  jobAds.addOperation(images);

  await scraper.scrape(root);

  fs.writeFile("./pages.json", JSON.stringify(pages), () => {}); //Produces a formatted JSON with all job ads.
})();

const sanitize = require("sanitize-filename"); //Using this npm module to sanitize file names.
const fs = require("fs");
const { Scraper, Root, OpenLinks } = require("nodejs-web-scraper");

(async () => {
  const config = {
    baseSiteUrl: `https://www.profesia.sk`,
    startUrl: `https://www.profesia.sk/praca/`,
    removeStyleAndScriptTags: false, //Telling the scraper NOT to remove style and script tags, cause i want it in my html files, for this example.
  };

  let directoryExists;

  const getPageHtml = (html, pageAddress) => {
    //Saving the HTML file, using the page address as a name.

    if (!directoryExists) {
      fs.mkdirSync("./html");
      directoryExists = true;
    }
    const name = sanitize(pageAddress);
    fs.writeFile(`./html/${name}.html`, html, () => {});
  };

  const scraper = new Scraper(config);

  const root = new Root({
    pagination: { queryString: "page_num", begin: 1, end: 100 },
  });

  const jobAds = new OpenLinks(".list-row h2 a", { getPageHtml }); //Opens every job ad, and calls a hook after every page is done.

  root.addOperation(jobAds);

  await scraper.scrape(root);
})();

const config = {
  baseSiteUrl: `https://www.some-content-site.com`,
  startUrl: `https://www.some-content-site.com/videos`,
  filePath: "./videos/",
  logPath: "./logs/",
};

const scraper = new Scraper(config);

const root = new Root();

const video = new DownloadContent("a.video", { contentType: "file" }); //The "contentType" makes it clear for the scraper that this is NOT an image(therefore the "href is used instead of "src").

const description = new CollectContent("h1").root.addOperation(video);
root.addOperation(description);

await scraper.scrape(root);

console.log(description.getData()); //You can call the "getData" method on every operation object, giving you the aggregated data collected by it.

  const getPageResponse = async (response) => {
        //Do something with response.data(the HTML content). No need to return anything.
    }

    const myDivs=[];

    const getElementContent = (content, pageAddress) => {

        myDivs.push(`myDiv content from page ${pageAddress} is ${content}...`)
    }

    const config = {
        baseSiteUrl: `https://www.nice-site`,
        startUrl: `https://www.nice-site/some-section`,
       }

    const scraper = new Scraper(config);

    const root = new Root();

    const articles = new OpenLinks('article a');

    const posts = new OpenLinks('.post a'{getPageResponse});//Is called after the HTML of a link was fetched, but before the children have been scraped. Is passed the response object of the page.

    const myDiv = new CollectContent('.myDiv',{getElementContent});//Will be called after every "myDiv" element is collected.

    root.addOperation(articles);
        articles.addOperation(myDiv);
    root.addOperation(posts);
        posts.addOperation(myDiv)

   await scraper.scrape(root);

const config = {
  baseSiteUrl: `https://www.nice-site`,
  startUrl: `https://www.nice-site/some-section`,
};

/**
 * Will be called for each node collected by cheerio, in the given operation(OpenLinks or DownloadContent)
 */
const condition = (cheerioNode) => {
  //Note that cheerioNode contains other useful methods, like html(), hasClass(), parent(), attr() and more.
  const text = cheerioNode.text().trim(); //Get the innerText of the <a> tag.
  if (text === "some text i am looking for") {
    //Even though many links might fit the querySelector, Only those that have this innerText,
    // will be "opened".
    return true;
  }
};

const scraper = new Scraper(config);

const root = new Root();

//Let's assume this page has many links with the same CSS class, but not all are what we need.
const linksToOpen = new OpenLinks("some-css-class-that-is-just-not-enough", {
  condition,
});

root.addOperation(linksToOpen);

await scraper.scrape(root);

const config = {
  baseSiteUrl: "", //Mandatory.If your site sits in a subfolder, provide the path WITHOUT it.
  startUrl: "", //Mandatory. The page from which the process begins.
  logPath: null, //Highly recommended.Will create a log for each scraping operation(object).
  cloneFiles: true, //If an image with the same name exists, a new file with a number appended to it is created. Otherwise. it's overwritten.
  removeStyleAndScriptTags: true, // Removes any <style> and <script> tags found on the page, in order to serve Cheerio with a light-weight string. change this ONLY if you have to.
  concurrency: 3, //Maximum concurrent requests.Highly recommended to keep it at 10 at most.
  maxRetries: 5, //Maximum number of retries of a failed request.
  delay: 200,
  timeout: 6000,
  filePath: null, //Needs to be provided only if a "downloadContent" operation is created.
  auth: null, //Can provide basic auth credentials(no clue what sites actually use it)
  headers: null, //Provide custom headers for the requests.
  proxy: null, //Use a proxy. Pass a full proxy URL, including the protocol and the port.
  showConsoleLogs: true, //Set to false, if you want to disable the messages
};

const root = new Root({
  pagination: { queryString: "page", begin: 1, end: 100 },
});

{
    pagination:{},//In case your root page is paginated.
    getPageObject:(pageObject,address)=>{},//Gets a formatted page object with all the data we choose in our scraping setup. Also gets an address argument.
    getPageHtml:(htmlString,pageAddress)=>{}//Get the entire html page, and also the page address. Called with each link opened by this OpenLinks object.
    getPageData:(cleanData)=>{}//Called after all data was collected by the root and its children.
    getPageResponse:(response)=>{}//Will be called after a link's html was fetched, but BEFORE the child operations are performed on it(like, collecting some data from it). Is passed the response object(a custom response object, that also contains the original node-fetch response). Notice that any modification to this object, might result in an unexpected behavior with the child operations of that page.
    getException:(error)=>{}//Get every exception thrown by Root.

}

{
    name:'some name',//Like every operation object, you can specify a name, for better clarity in the logs.
    pagination:{},//Look at the pagination API for more details.
    condition:(cheerioNode)=>{},//Use this hook to add additional filter to the nodes that were received by the querySelector. Return true to include, falsy to exclude.
    getPageObject:(pageObject,address)=>{},//Gets a formatted page object with all the data we choose in our scraping setup. Also gets an address argument.
    getPageHtml:(htmlString,pageAddress)=>{}//Get the entire html page, and also the page address. Called with each link opened by this OpenLinks object.
    getElementList:(elementList)=>{},//Is called each time an element list is created. In the case of OpenLinks, will happen with each list of anchor tags that it collects. Those elements all have Cheerio methods available to them.
    getPageData:(cleanData)=>{}//Called after all data was collected from a link, opened by this object.(if a given page has 10 links, it will be called 10 times, with the child data).
    getPageResponse:(response)=>{}//Will be called after a link's html was fetched, but BEFORE the child operations are performed on it(like, collecting some data from it). Is passed the response object(a custom response object, that also contains the original node-fetch response). Notice that any modification to this object, might result in an unexpected behavior with the child operations of that page.
    getException:(error)=>{}//Get every exception throw by this openLinks operation, even if this was later repeated successfully.
    slice:[start,end]//You can define a certain range of elements from the node list.Also possible to pass just a number, instead of an array, if you only want to specify the start. This uses the Cheerio/Jquery slice method.
}

{
    name:'some name',
    contentType:'text',//Either 'text' or 'html'. Default is text.
    shouldTrim:true,//Default is true. Applies JS String.trim() method.
    getElementList:(elementList,pageAddress)=>{},
    getElementContent:(elementContentString,pageAddress)=>{}//Called with each element collected.
    getAllItems: (items, address)=>{}//Called after an entire page has its elements collected.
    slice:[start,end]
}

{
    name:'some name',
    contentType:'image',//Either 'image' or 'file'. Default is image.
    alternativeSrc:['first-alternative','second-alternative']//Provide alternative attributes to be used as the src. Will only be invoked,
    //If the "src" attribute is undefined or is a dataUrl. If no matching alternative is found, the dataUrl is used.
    condition:(cheerioNode)=>{},//Use this hook to add additional filter to the nodes that were received by the querySelector. Return true to include, falsy to exclude.
    getElementList:(elementList)=>{},
    getException:(error)=>{}//Get every exception throw by this downloadContent operation, even if this was later repeated successfully.
    filePath:'./somePath',//Overrides the global filePath passed to the Scraper config.
    slice:[start,end]
}

    //If a site uses a queryString for pagination, this is how it's done:

    const productPages = new openLinks('a.product'{ pagination: { queryString: 'page_num', begin: 1, end: 1000 } });//You need to specify the query string that the site uses for pagination, and the page range you're interested in.

    //If the site uses some kind of offset(like Google search results), instead of just incrementing by one, you can do it this way:

    { pagination: { queryString: 'page_num', begin: 1, end: 100,offset:10 } }

    //If the site uses routing-based pagination:

    { pagination: { routingString: '/', begin: 1, end: 100 } }