[Answered ]-Scrape data from paginated contents

I used selenium and phantom js to do this.I used window.scrollTo to get the entire page and this worked for me

def handle(self, *args, **options):

       driver = webdriver.PhantomJS()

       driver.get("http://example.com")
       time.sleep(3)

        # elem = driver.find_element_by_tag_name("body")
        driver.set_window_size(1024, 768)

        no_of_pagedowns = 20

        while no_of_pagedowns:
            # elem.send_keys(Keys.PAGE_DOWN)
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(1)
            no_of_pagedowns-=1

        post_elems = driver.find_elements_by_class_name("jcn")
        driver.save_screenshot('testing.png')
        for post in post_elems:
            ###Operations to be done
    driver.close()