DaveM@23: #!/usr/bin/env python
DaveM@23: 
DaveM@23: 
DaveM@23: import dParse as dp
DaveM@23: import synastry as syn
DaveM@23: # import requests
DaveM@29: import localRequests as lr
DaveM@23: import csv
DaveM@23: import pdb
DaveM@23: import os
DaveM@23: import pickle
DaveM@23: import sys
DaveM@23: from bs4 import BeautifulSoup
DaveM@23: 
DaveM@23: def makePeople(filename):
DaveM@23: 	stream = csv.DictReader(open(filename,'rb'))
DaveM@23: 	dictList = []
DaveM@23: 	people = []
DaveM@23: 	for line in stream:
DaveM@23: 		thisPerson = syn.Person(dp.regulateData(line))
DaveM@23: 		people.append(thisPerson)
DaveM@23: 		# pdb.set_trace()
DaveM@23: 	return people
DaveM@23: 
DaveM@23: def uniquify(itemList):
DaveM@23: 	keyDict = {}
DaveM@23: 	for item in itemList:
DaveM@23: 		keyDict[item] = 1
DaveM@23: 	return sorted(keyDict.keys())
DaveM@23: 
DaveM@23: def outputPeople(filename,people):
DaveM@23: 	with open(filename, "wb") as csv_file:
DaveM@23: 		dictKeys = []
DaveM@23: 		for person in people:
DaveM@23: 			if person.issue is None:
DaveM@23: 				person.horiscope.makeAllAspectTreple()
DaveM@23: 				dictKeys += person.horiscope.aspectTreple.keys()
DaveM@23: 		dictKeys = uniquify(dictKeys)
DaveM@23: 		writer = csv.DictWriter(csv_file, ['id','score']+dictKeys)
DaveM@23: 		writer.writeheader()
DaveM@23: 		for person in people:
DaveM@23: 			if person.issue is None:
DaveM@23: 				tempDict = {'id':person.id,'score':person.score}
DaveM@23: 				tempDict.update(person.horiscope.aspectTreple)
DaveM@23: 				# pdb.set_trace()
DaveM@23: 				writer.writerow(tempDict)
DaveM@23: 
DaveM@23: def outputScores(filename,people):
DaveM@23: 	with open(filename, "wb") as csv_file:
DaveM@23: 		csv_file.write('Person ID, Score \n')
DaveM@23: 		for person in people:
DaveM@23: 			if person.issue is None:
DaveM@23: 				csv_file.write(str(person.id)+','+str(person.score)+'\n')
DaveM@23: 			else:
DaveM@23: 				csv_file.write(str(person.id)+','+str(person.issue)+'\n')
DaveM@23: 
DaveM@29: def outputDescriptions(filename,people):
DaveM@29: 	with open(filename, "wb") as csv_file:
DaveM@29: 		for person in people:
DaveM@29: 			if person.issue is None:
DaveM@31: 				csv_file.write(str(person.id)+', '+str(person.score)+'\n'+''.join(person.description)+'\n\n')
DaveM@29: 			else:
DaveM@31: 				csv_file.write(str(person.id)+', '+str(person.issue)+'\n\n')
DaveM@29: 
DaveM@23: def outputIssues(filename,people):
DaveM@23: 	with open(filename, "wb") as csv_file:
DaveM@23: 		csv_file.write('Person ID, Issue \n')
DaveM@23: 		for person in people:
DaveM@23: 			if person.issue is not None:
DaveM@23: 				csv_file.write(str(person.id)+','+str(person.issue)+'\n')
DaveM@23: 
DaveM@23: # def requestURL(url,payload):
DaveM@23: # 	r = requests.get(url, params=payload)
DaveM@23: # 	time.sleep(5)
DaveM@23: # 	return r
DaveM@23: 
DaveM@23: # def loadPickUpdate(filename):
DaveM@23: 
DaveM@23: def loadPick(filename):
DaveM@23: 	with open(filename, 'rb') as handle:
DaveM@23: 		b = pickle.load(handle)
DaveM@23: 	return b
DaveM@23: 
DaveM@23: def savePick(filename,data):
DaveM@23: 	with open(filename, 'wb') as handle:
DaveM@23: 		pickle.dump(data,handle)
DaveM@23: 
DaveM@23: def parseHoriscope(people,saveFile):
DaveM@23: 	rawData = {}
DaveM@23: 	for person in people:
DaveM@23: 		issue = person.identifyIssues()
DaveM@23: 		if issue is not None:
DaveM@23: 			print 'SKIPPING person '+ person.id + ' error with ' + issue 
DaveM@23: 		else:
DaveM@24: 			# print 'parsing person '+ person.id
DaveM@23: 			if person.resp is None:
DaveM@23: 				print 'Posting Request for person '+ person.id
DaveM@23: 				person.makePayload()
DaveM@23: 				rawData[person.id] = person.requestURL()
DaveM@29: 
DaveM@24: 			# print 'parsing person '+ person.id
DaveM@29: 			if person.resp is None:
DaveM@29: 				print 'SKIPPING person '+ person.id + ' error with unknown error'
DaveM@29: 			else:
DaveM@29: 				person.horiscope = person.parsePage()
DaveM@23: 			# person.horiscope.printPositions()
DaveM@23: 		if saveFile is not None:
DaveM@28: 			# savePick('raw_'+saveFile,rawData)
DaveM@23: 			savePick(saveFile,people)
DaveM@23: 
DaveM@23: def parseHoriscopeRaw(people,saveFile):
DaveM@23: 	rawData = loadPick('raw_'+saveFile)
DaveM@23: 	for person in people:
DaveM@23: 		issue = person.identifyIssues()
DaveM@23: 		if issue is not None:
DaveM@23: 			print 'SKIPPING person '+ person.id + ' error with ' + issue 
DaveM@23: 		else:
DaveM@24: 			# print 'parsing person '+ person.id
DaveM@23: 			if person.resp is None:
DaveM@24: 				print 'Reading Request for person '+ person.id
DaveM@23: 				person.makePayload()
DaveM@23: 				person.resp = rawData[person.id]
DaveM@24: 			# print 'parsing person '+ person.id
DaveM@23: 			person.horiscope = person.parsePage()
DaveM@23: 			# person.horiscope.printPositions()
DaveM@23: 		if saveFile is not None:
DaveM@23: 			savePick(saveFile,people)
DaveM@23: 
DaveM@23: def _main():
DaveM@29: 	resetAll = False
DaveM@29: 	if resetAll:
DaveM@29: 		for item in os.listdir('.'):
DaveM@29: 			if item.endswith(".pick"):
DaveM@29: 				os.remove(item)
DaveM@31: 	# os.remove('outData_collect.pick')
DaveM@23: 	pickFile = 'outData.pick'
DaveM@23: 	if not os.path.exists(pickFile):
DaveM@23: 		people = makePeople('individuals.csv')
DaveM@23: 		savePick(pickFile,people)
DaveM@23: 	else:
DaveM@23: 		people = loadPick(pickFile)
DaveM@23: 	parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
DaveM@23: 	if not os.path.exists(parseSaveFile):
DaveM@27: 		# if os.path.exists('raw_'+parseSaveFile):
DaveM@27: 		# 	parseHoriscopeRaw(people,parseSaveFile)
DaveM@27: 		# else:
DaveM@27: 		parseHoriscope(people,parseSaveFile)
DaveM@23: 	else:
DaveM@23: 		people = loadPick(parseSaveFile)
DaveM@23: 	# if not os.path.exists('fullResults.pick'):
DaveM@23: 	if not os.path.exists('fullResults.pick'):
DaveM@23: 		comp = syn.compatibility()
DaveM@23: 		comp.parseCompatRules('compatibilityRules.csv')
DaveM@23: 		for person in people:
DaveM@23: 			print person.id
DaveM@23: 			person.score = None
DaveM@23: 			if person.issue is None:
DaveM@29: 				person.score,person.description = comp.describeCompatibility(person.horiscope)
DaveM@23: 				if person.score is None:
DaveM@23: 					person.issue = 'None Planet Locations'
DaveM@23: 					# pdb.set_trace()
DaveM@23: 					f = open('issues/'+str(person.id)+'.html','w')
DaveM@23: 					f.write(person.resp.content)
DaveM@23: 					f.close()
DaveM@23: 		savePick('fullResults.pick',people)
DaveM@23: 	else:
DaveM@23: 		people = loadPick('fullResults.pick')
DaveM@23: 	outputPeople('fullResult.csv',people)
DaveM@23: 	outputScores('scores.csv',people)
DaveM@31: 	outputDescriptions('description.txt',people)
DaveM@23: 	outputIssues('issues.csv',people)
DaveM@23: 
DaveM@23: if __name__ == "__main__":
DaveM@23: 	_main()