comparison V4/runme.py @ 14:a0c217ee4168

current edits
author DaveM
date Sun, 04 Mar 2018 15:03:15 +0000
parents b253748dbb11
children 50a95089414d
comparison
equal deleted inserted replaced
13:b253748dbb11 14:a0c217ee4168
12 import pickle 12 import pickle
13 from HTMLParser import HTMLParser 13 from HTMLParser import HTMLParser
14 # from lxml import html 14 # from lxml import html
15 from bs4 import BeautifulSoup 15 from bs4 import BeautifulSoup
16 16
17 def parsePage(horiscope, resp): 17 def parsePage(resp):
18 horiscope = syn.planetPositions() 18 horiscope = syn.planetPositions()
19 soup = BeautifulSoup(resp.content, 'lxml') 19 soup = BeautifulSoup(resp.content, 'lxml')
20 tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'}) 20 tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'})
21 for cell in tcCell: 21 for cell in tcCell:
22 divList = cell.find_all('div') 22 divList = cell.find_all('div')
23 for i in range(len(divList)): 23 for i in range(len(divList)):
24 planetName = divList[i].getText().lower() 24 planetName = divList[i].getText().lower()
25 if planetName in planetPositions.planetNames: 25 if planetName in planetPositions.planetNames:
26 horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText()) 26 horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText())
27 return horiscope
27 28
28 def makePeople(filename): 29 def makePeople(filename):
29 stream = csv.DictReader(open(filename,'rb')) 30 stream = csv.DictReader(open(filename,'rb'))
30 dictList = [] 31 dictList = []
31 people = [] 32 people = []
97 parseTries = 3 98 parseTries = 3
98 while parseTries > 0: 99 while parseTries > 0:
99 try: 100 try:
100 person.makePayload() 101 person.makePayload()
101 resp = requestURL(person.url,person.payload) 102 resp = requestURL(person.url,person.payload)
102 parsePage(person.horiscope,resp) 103 person.horiscope = parsePage(resp)
103 pdb.set_trace() 104 pdb.set_trace()
104 parseTries = 0 105 parseTries = 0
105 except: 106 except:
106 print sys.exc_info()[0] 107 print sys.exc_info()[0]
107 parseTries -= 1 108 parseTries -= 1