DaveM@13: #!/usr/bin/env python
DaveM@15: 
DaveM@15: 
DaveM@13: import dParse as dp
DaveM@13: import synastry as syn
DaveM@19: # import requests
DaveM@19: # import time
DaveM@13: import csv
DaveM@13: import pdb
DaveM@13: import os
DaveM@13: import pickle
DaveM@15: import sys
DaveM@13: from bs4 import BeautifulSoup
DaveM@13: 
DaveM@19: # def parsePage(resp):
DaveM@19: # 	gotLocation = 0
DaveM@19: # 	horiscope = syn.planetPositions()
DaveM@19: # 	soup = BeautifulSoup(resp.content, 'lxml')
DaveM@19: # 	tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'})
DaveM@19: # 	for cell in tcCell:
DaveM@19: # 		if "Planets in partner's house" in cell.get_text():
DaveM@19: # 			gotLocation = 1
DaveM@19: # 		divList = cell.find_all('div')
DaveM@19: # 		for i in range(len(divList)):
DaveM@19: # 			planetName = divList[i].getText().lower().strip().replace(':','').split(' ')[0]
DaveM@19: # 			if planetName in syn.planetPositions.planetNames:
DaveM@19: # 				if gotLocation and not '/' in planetName:
DaveM@19: # 					horiscope.planets[planetName].setHouse(divList[i+2].getText(),divList[i+4].getText())
DaveM@19: # 				else:
DaveM@19: # 					horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText())
DaveM@19: # 	return horiscope
DaveM@13: 
DaveM@13: def makePeople(filename):
DaveM@13: 	stream = csv.DictReader(open(filename,'rb'))
DaveM@13: 	dictList = []
DaveM@13: 	people = []
DaveM@13: 	for line in stream:
DaveM@13: 		thisPerson = syn.Person(dp.regulateData(line))
DaveM@13: 		people.append(thisPerson)
DaveM@13: 		# pdb.set_trace()
DaveM@13: 	return people
DaveM@13: 
DaveM@19: def uniquify(itemList):
DaveM@19: 	keyDict = {}
DaveM@19: 	for item in itemList:
DaveM@19: 		keyDict[item] = 1
DaveM@22: 	return sorted(keyDict.keys())
DaveM@19: 
DaveM@19: def outputPeople(filename,people):
DaveM@19: 	with open(filename, "wb") as csv_file:
DaveM@19: 		dictKeys = []
DaveM@19: 		for person in people:
DaveM@19: 			if person.issue is None:
DaveM@22: 				person.horiscope.makeAllAspectTreple()
DaveM@22: 				dictKeys += person.horiscope.aspectTreple.keys()
DaveM@19: 		dictKeys = uniquify(dictKeys)
DaveM@22: 		writer = csv.DictWriter(csv_file, ['id','score']+dictKeys)
DaveM@19: 		writer.writeheader()
DaveM@19: 		for person in people:
DaveM@19: 			if person.issue is None:
DaveM@22: 				tempDict = {'id':person.id,'score':person.score}
DaveM@22: 				tempDict.update(person.horiscope.aspectTreple)
DaveM@19: 				# pdb.set_trace()
DaveM@19: 				writer.writerow(tempDict)
DaveM@19: 
DaveM@19: def outputScores(filename,people):
DaveM@19: 	with open(filename, "wb") as csv_file:
DaveM@19: 		csv_file.write('Person ID, Score \n')
DaveM@19: 		for person in people:
DaveM@19: 			if person.issue is None:
DaveM@19: 				csv_file.write(str(person.id)+','+str(person.score)+'\n')
DaveM@19: 			else:
DaveM@19: 				csv_file.write(str(person.id)+','+str(person.issue)+'\n')
DaveM@19: 
DaveM@19: def outputIssues(filename,people):
DaveM@19: 	with open(filename, "wb") as csv_file:
DaveM@19: 		csv_file.write('Person ID, Issue \n')
DaveM@19: 		for person in people:
DaveM@19: 			if person.issue is not None:
DaveM@19: 				csv_file.write(str(person.id)+','+str(person.issue)+'\n')
DaveM@19: 
DaveM@19: # def requestURL(url,payload):
DaveM@19: # 	r = requests.get(url, params=payload)
DaveM@19: # 	time.sleep(5)
DaveM@19: # 	return r
DaveM@13: 
DaveM@23: # def loadPickUpdate(filename):
DaveM@22: 
DaveM@13: def loadPick(filename):
DaveM@13: 	with open(filename, 'rb') as handle:
DaveM@13: 		b = pickle.load(handle)
DaveM@13: 	return b
DaveM@13: 
DaveM@13: def savePick(filename,data):
DaveM@13: 	with open(filename, 'wb') as handle:
DaveM@13: 		pickle.dump(data,handle)
DaveM@13: 
DaveM@13: def parseHoriscope(people,saveFile):
DaveM@13: 	for person in people:
DaveM@15: 		issue = person.identifyIssues()
DaveM@15: 		if issue is not None:
DaveM@15: 			print 'SKIPPING person '+ person.id + ' error with ' + issue 
DaveM@13: 		else:
DaveM@13: 			print 'parsing person '+ person.id
DaveM@22: 			if person.resp is None:
DaveM@22: 				print 'Posting Request for person '+ person.id
DaveM@22: 				person.makePayload()
DaveM@22: 				person.requestURL()
DaveM@22: 			print 'parsing person '+ person.id
DaveM@19: 			person.horiscope = person.parsePage()
DaveM@15: 			# person.horiscope.printPositions()
DaveM@15: 		if saveFile is not None:
DaveM@15: 				savePick(saveFile,people)
DaveM@13: 
DaveM@16: # def testMain():
DaveM@16: # 	pickFile = 'outData.pick'
DaveM@16: # 	restartDataFile = 0
DaveM@16: # 	if(restartDataFile):
DaveM@16: # 		# people = makePeople('individuals.csv')
DaveM@16: # 		# savePick(pickFile,people)
DaveM@16: # 		people = loadPick(pickFile)
DaveM@16: # 		parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
DaveM@16: # 		parseHoriscope(people,parseSaveFile)
DaveM@16: # 	else:
DaveM@16: # 		people = loadPick('outData_collect.pick')
DaveM@16: # 	comp = syn.compatibility()
DaveM@16: # 	comp.parseCompatRules('compatibilityRules.csv')
DaveM@16: # 	for person in people:
DaveM@16: # 		if person.issue is None:
DaveM@16: # 			person.score = comp.calcCompatibility(person.horiscope)
DaveM@16: # 			if person.score is None:
DaveM@16: # 				person.issue = 'None Planet Locations'
DaveM@16: # 			else:
DaveM@16: # 				print person.id,person.score
DaveM@16: # 	pdb.set_trace()
DaveM@13: 
DaveM@13: def _main():
DaveM@13: 	pickFile = 'outData.pick'
DaveM@13: 	if not os.path.exists(pickFile):
DaveM@13: 		people = makePeople('individuals.csv')
DaveM@13: 		savePick(pickFile,people)
DaveM@13: 	else:
DaveM@13: 		people = loadPick(pickFile)
DaveM@13: 	parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
DaveM@16: 	if not os.path.exists(parseSaveFile):
DaveM@16: 		parseHoriscope(people,parseSaveFile)
DaveM@16: 	else:
DaveM@16: 		people = loadPick(parseSaveFile)
DaveM@23: 	# if not os.path.exists('fullResults.pick'):
DaveM@23: 	if os.path.exists('fullResults.pick'):
DaveM@19: 		comp = syn.compatibility()
DaveM@19: 		comp.parseCompatRules('compatibilityRules.csv')
DaveM@19: 		for person in people:
DaveM@23: 			print person.id
DaveM@19: 			person.score = None
DaveM@19: 			if person.issue is None:
DaveM@19: 				person.score = comp.calcCompatibility(person.horiscope)
DaveM@19: 				if person.score is None:
DaveM@19: 					person.issue = 'None Planet Locations'
DaveM@19: 					# pdb.set_trace()
DaveM@19: 					f = open('issues/'+str(person.id)+'.html','w')
DaveM@19: 					f.write(person.resp.content)
DaveM@19: 					f.close()
DaveM@19: 		savePick('fullResults.pick',people)
DaveM@19: 	else:
DaveM@19: 		people = loadPick('fullResults.pick')
DaveM@21: 	outputPeople('fullResult.csv',people)
DaveM@21: 	outputScores('scores.csv',people)
DaveM@21: 	outputIssues('issues.csv',people)
DaveM@13: 
DaveM@13: if __name__ == "__main__":
DaveM@16: 	_main()