DaveM@2: #!/usr/bin/env python
DaveM@3: import dParse as dp
DaveM@2: import requests
DaveM@2: import re
DaveM@2: import time
DaveM@2: import csv
DaveM@2: import random
DaveM@2: from HTMLParser import HTMLParser
DaveM@2: # from lxml import html
DaveM@2: from bs4 import BeautifulSoup
DaveM@2: 
DaveM@2: def parsePage(resp):
DaveM@2: 	soup = BeautifulSoup(resp.content, 'lxml')
DaveM@2: 	tcCell = soup.find_all('div', attrs={'class':'tc'})
DaveM@2: 	for cell in tcCell:
DaveM@2: 		person = dict()
DaveM@2: 		tableCell = cell.find_all('td');
DaveM@2: 		if len(tableCell) > 2:
DaveM@2: 			C = tableCell[0].strong.contents[0].encode('utf-8')
DaveM@2: 			D = tableCell[2].strong.contents[0].encode('utf-8')
DaveM@2: 			print (C,D)
DaveM@2: 			A = re.search("\/>(.*)<br/>.*\(([0-9]*)\\xc2\\xb0([0-9]*)(.*)\)",str(tableCell[1]))
DaveM@2: 			# A0 = A.group(1)
DaveM@2: 			# A1 = A.group(2).split('\xc2\xb0')[0]
DaveM@2: 			# A2 = A.group(2).split('\xc2\xb0')[1].split('\xe2')[0]
DaveM@2: 			print (A.group(1),A.group(2),A.group(3))
DaveM@2: 			person[(C,D)] = (A.group(1),A.group(2),A.group(3))
DaveM@2: 		return person
DaveM@2: 
DaveM@2: 
DaveM@2: def setURL(p):
DaveM@2: 	""" 
DaveM@2: 	Code impacting factors into URL
DaveM@2: 	IMPACTING FACTORS
DaveM@2: 	Date of Birth
DaveM@2: 	Birth Time
DaveM@2: 	Country of birth
DaveM@2: 	City of birth (And state of birth)
DaveM@2: 	"""
DaveM@2: 	## For some reason we need to post men first then women.
DaveM@2: 	# url = "https://horoscopes.astro-seek.com/calculate-love-compatibility/?send_calculation=1&muz_narozeni_den=1&muz_narozeni_mesic=1&muz_narozeni_rok=1970&muz_narozeni_hodina=00&muz_narozeni_minuta=00&muz_narozeni_city=London%2C+United+Kingdom&muz_narozeni_mesto_hidden=London&muz_narozeni_stat_hidden=GB&muz_narozeni_podstat_kratky_hidden=England&muz_narozeni_podstat_hidden=England&muz_narozeni_podstat2_kratky_hidden=Greater+London&muz_narozeni_podstat3_kratky_hidden=undefined&muz_narozeni_input_hidden=&muz_narozeni_sirka_stupne=51&muz_narozeni_sirka_minuty=30&muz_narozeni_sirka_smer=0&muz_narozeni_delka_stupne=0&muz_narozeni_delka_minuty=8&muz_narozeni_delka_smer=1&muz_narozeni_timezone_form=auto&muz_narozeni_timezone_dst_form=auto&send_calculation=1&zena_narozeni_den=1&zena_narozeni_mesic=1&zena_narozeni_rok=1970&zena_narozeni_hodina=00&zena_narozeni_minuta=00&zena_narozeni_city=Berlin%2C+Germany&zena_narozeni_mesto_hidden=Berlin&zena_narozeni_stat_hidden=DE&zena_narozeni_podstat_kratky_hidden=Berlin&zena_narozeni_podstat_hidden=Berlin&zena_narozeni_podstat2_kratky_hidden=undefined&zena_narozeni_podstat3_kratky_hidden=undefined&zena_narozeni_input_hidden=&zena_narozeni_sirka_stupne=52&zena_narozeni_sirka_minuty=31&zena_narozeni_sirka_smer=0&zena_narozeni_delka_stupne=13&zena_narozeni_delka_minuty=24&zena_narozeni_delka_smer=0&zena_narozeni_timezone_form=auto&zena_narozeni_timezone_dst_form=auto&switch_interpretations=0&house_system=placidus&uhel_orbis=#tabs_redraw"
DaveM@2: 	# payload = {'muz_narozeni_den':'1','muz_narozeni_mesic':'1','muz_narozeni_rok':'1970'}
DaveM@2: 	url = 'https://horoscopes.astro-seek.com/calculate-love-compatibility/'
DaveM@2: 	mDay = random.randint(1,29)
DaveM@2: 	mMonth = random.randint(1,12)
DaveM@2: 	mYear = random.randint(1,100)+1917
DaveM@2: 	fDay = random.randint(1,29)
DaveM@2: 	fMonth = random.randint(1,12)
DaveM@2: 	fYear = random.randint(1,100)+1917
DaveM@2: 
DaveM@2: 	payload = {'?send_calculation':'1','muz_narozeni_den':mDay,'muz_narozeni_mesic':mMonth,'muz_narozeni_rok':mYear,'zena_narozeni_den':fDay,'zena_narozeni_mesic':fMonth,'zena_narozeni_rok':fYear}
DaveM@2: 	return (url,payload)
DaveM@2: 
DaveM@2: def requestURL(url,payload):
DaveM@2: 	r = requests.get(url, params=payload)
DaveM@2: 	time.sleep(5)
DaveM@2: 	return r
DaveM@2: 
DaveM@4: # def parseCSV(filename):
DaveM@4: # 	stream = csv.DictReader(open(filename,'rb'))
DaveM@4: # 	dictList = []
DaveM@4: # 	for line in stream:
DaveM@4: # 		dictList.append(regulateData(line))
DaveM@3: 
DaveM@4: # 	# dictList = headerParse(dictList)
DaveM@4: # 	# dictList = validateData(dictList)
DaveM@4: # 	return dictList
DaveM@2: 
DaveM@2: def printToFile(filename,data):
DaveM@2: 	keys = data[0].keys()
DaveM@2: 	with open(filename,'w') as stream:
DaveM@2: 		dict_writer = csv.DictWriter(stream, keys)
DaveM@2: 		dict_writer.writeheader()
DaveM@2: 		dict_writer.writerows(data)
DaveM@2: 
DaveM@3: def testMain():
DaveM@4: 	people = dp.parseCSV('individuals.csv')
DaveM@3: 
DaveM@3: def _main():
DaveM@2: 	# people = dict()
DaveM@4: 	people = dp.parseCSV('individuals.csv')
DaveM@2: 	horiscopeList = []
DaveM@2: 	# people = [1,2,3,4,5]
DaveM@2: 	for person in people:
DaveM@2: 		print 'parsing person '+ person['ID']
DaveM@2: 		url,payload = setURL('')
DaveM@2: 		resp = requestURL(url,payload)
DaveM@2: 		
DaveM@2: 		person['horiscope'] = parsePage(resp)
DaveM@2: 		horiscopeList.append(person)
DaveM@2: 	print horiscopeList
DaveM@2: 	printToFile('outputdata.csv',horiscopeList)
DaveM@2: 
DaveM@2: if __name__ == "__main__":
DaveM@3: 	testMain()