annotate V5/runme.py @ 23:11d4e438045e

make version 5

#!/usr/bin/env python


import dParse as dp
import synastry as syn
# import requests
# import time
import csv
import pdb
import os
import pickle
import sys
from bs4 import BeautifulSoup

# def parsePage(resp):
# 	gotLocation = 0
# 	horiscope = syn.planetPositions()
# 	soup = BeautifulSoup(resp.content, 'lxml')
# 	tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'})
# 	for cell in tcCell:
# 		if "Planets in partner's house" in cell.get_text():
# 			gotLocation = 1
# 		divList = cell.find_all('div')
# 		for i in range(len(divList)):
# 			planetName = divList[i].getText().lower().strip().replace(':','').split(' ')[0]
# 			if planetName in syn.planetPositions.planetNames:
# 				if gotLocation and not '/' in planetName:
# 					horiscope.planets[planetName].setHouse(divList[i+2].getText(),divList[i+4].getText())
# 				else:
# 					horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText())
# 	return horiscope

def makePeople(filename):
	stream = csv.DictReader(open(filename,'rb'))
	dictList = []
	people = []
	for line in stream:
		thisPerson = syn.Person(dp.regulateData(line))
		people.append(thisPerson)
		# pdb.set_trace()
	return people

def uniquify(itemList):
	keyDict = {}
	for item in itemList:
		keyDict[item] = 1
	return sorted(keyDict.keys())

def outputPeople(filename,people):
	with open(filename, "wb") as csv_file:
		dictKeys = []
		for person in people:
			if person.issue is None:
				person.horiscope.makeAllAspectTreple()
				dictKeys += person.horiscope.aspectTreple.keys()
		dictKeys = uniquify(dictKeys)
		writer = csv.DictWriter(csv_file, ['id','score']+dictKeys)
		writer.writeheader()
		for person in people:
			if person.issue is None:
				tempDict = {'id':person.id,'score':person.score}
				tempDict.update(person.horiscope.aspectTreple)
				# pdb.set_trace()
				writer.writerow(tempDict)

def outputScores(filename,people):
	with open(filename, "wb") as csv_file:
		csv_file.write('Person ID, Score \n')
		for person in people:
			if person.issue is None:
				csv_file.write(str(person.id)+','+str(person.score)+'\n')
			else:
				csv_file.write(str(person.id)+','+str(person.issue)+'\n')

def outputIssues(filename,people):
	with open(filename, "wb") as csv_file:
		csv_file.write('Person ID, Issue \n')
		for person in people:
			if person.issue is not None:
				csv_file.write(str(person.id)+','+str(person.issue)+'\n')

# def requestURL(url,payload):
# 	r = requests.get(url, params=payload)
# 	time.sleep(5)
# 	return r

# def loadPickUpdate(filename):

def loadPick(filename):
	with open(filename, 'rb') as handle:
		b = pickle.load(handle)
	return b

def savePick(filename,data):
	with open(filename, 'wb') as handle:
		pickle.dump(data,handle)

def parseHoriscope(people,saveFile):
	rawData = {}
	for person in people:
		issue = person.identifyIssues()
		if issue is not None:
			print 'SKIPPING person '+ person.id + ' error with ' + issue 
		else:
			print 'parsing person '+ person.id
			if person.resp is None:
				print 'Posting Request for person '+ person.id
				person.makePayload()
				rawData[person.id] = person.requestURL()
			print 'parsing person '+ person.id
			person.horiscope = person.parsePage()
			# person.horiscope.printPositions()
		if saveFile is not None:
			savePick('raw_'+saveFile,rawData)
			savePick(saveFile,people)

def parseHoriscopeRaw(people,saveFile):
	rawData = loadPick('raw_'+saveFile)
	for person in people:
		issue = person.identifyIssues()
		if issue is not None:
			print 'SKIPPING person '+ person.id + ' error with ' + issue 
		else:
			print 'parsing person '+ person.id
			if person.resp is None:
				print 'Posting Request for person '+ person.id
				person.makePayload()
				person.resp = rawData[person.id]
			print 'parsing person '+ person.id
			person.horiscope = person.parsePage()
			# person.horiscope.printPositions()
		if saveFile is not None:
			savePick(saveFile,people)

# def testMain():
# 	pickFile = 'outData.pick'
# 	restartDataFile = 0
# 	if(restartDataFile):
# 		# people = makePeople('individuals.csv')
# 		# savePick(pickFile,people)
# 		people = loadPick(pickFile)
# 		parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
# 		parseHoriscope(people,parseSaveFile)
# 	else:
# 		people = loadPick('outData_collect.pick')
# 	comp = syn.compatibility()
# 	comp.parseCompatRules('compatibilityRules.csv')
# 	for person in people:
# 		if person.issue is None:
# 			person.score = comp.calcCompatibility(person.horiscope)
# 			if person.score is None:
# 				person.issue = 'None Planet Locations'
# 			else:
# 				print person.id,person.score
# 	pdb.set_trace()

def _main():
	pickFile = 'outData.pick'
	if not os.path.exists(pickFile):
		people = makePeople('individuals.csv')
		savePick(pickFile,people)
	else:
		people = loadPick(pickFile)
	parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
	if not os.path.exists(parseSaveFile):
		if os.path.exists('raw_'+parseSaveFile):
			parseHoriscopeRaw(people,parseSaveFile)
		else:
			parseHoriscope(people,parseSaveFile)
	else:
		people = loadPick(parseSaveFile)
	# if not os.path.exists('fullResults.pick'):
	if not os.path.exists('fullResults.pick'):
		comp = syn.compatibility()
		comp.parseCompatRules('compatibilityRules.csv')
		for person in people:
			print person.id
			person.score = None
			if person.issue is None:
				person.score = comp.calcCompatibility(person.horiscope)
				if person.score is None:
					person.issue = 'None Planet Locations'
					# pdb.set_trace()
					f = open('issues/'+str(person.id)+'.html','w')
					f.write(person.resp.content)
					f.close()
		savePick('fullResults.pick',people)
	else:
		people = loadPick('fullResults.pick')
	outputPeople('fullResult.csv',people)
	outputScores('scores.csv',people)
	outputIssues('issues.csv',people)

if __name__ == "__main__":
	_main()