diff V5/runme.py @ 23:11d4e438045e

make version 5

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/V5/runme.py	Mon Apr 09 15:07:21 2018 +0100
@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+
+
+import dParse as dp
+import synastry as syn
+# import requests
+# import time
+import csv
+import pdb
+import os
+import pickle
+import sys
+from bs4 import BeautifulSoup
+
+# def parsePage(resp):
+# 	gotLocation = 0
+# 	horiscope = syn.planetPositions()
+# 	soup = BeautifulSoup(resp.content, 'lxml')
+# 	tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'})
+# 	for cell in tcCell:
+# 		if "Planets in partner's house" in cell.get_text():
+# 			gotLocation = 1
+# 		divList = cell.find_all('div')
+# 		for i in range(len(divList)):
+# 			planetName = divList[i].getText().lower().strip().replace(':','').split(' ')[0]
+# 			if planetName in syn.planetPositions.planetNames:
+# 				if gotLocation and not '/' in planetName:
+# 					horiscope.planets[planetName].setHouse(divList[i+2].getText(),divList[i+4].getText())
+# 				else:
+# 					horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText())
+# 	return horiscope
+
+def makePeople(filename):
+	stream = csv.DictReader(open(filename,'rb'))
+	dictList = []
+	people = []
+	for line in stream:
+		thisPerson = syn.Person(dp.regulateData(line))
+		people.append(thisPerson)
+		# pdb.set_trace()
+	return people
+
+def uniquify(itemList):
+	keyDict = {}
+	for item in itemList:
+		keyDict[item] = 1
+	return sorted(keyDict.keys())
+
+def outputPeople(filename,people):
+	with open(filename, "wb") as csv_file:
+		dictKeys = []
+		for person in people:
+			if person.issue is None:
+				person.horiscope.makeAllAspectTreple()
+				dictKeys += person.horiscope.aspectTreple.keys()
+		dictKeys = uniquify(dictKeys)
+		writer = csv.DictWriter(csv_file, ['id','score']+dictKeys)
+		writer.writeheader()
+		for person in people:
+			if person.issue is None:
+				tempDict = {'id':person.id,'score':person.score}
+				tempDict.update(person.horiscope.aspectTreple)
+				# pdb.set_trace()
+				writer.writerow(tempDict)
+
+def outputScores(filename,people):
+	with open(filename, "wb") as csv_file:
+		csv_file.write('Person ID, Score \n')
+		for person in people:
+			if person.issue is None:
+				csv_file.write(str(person.id)+','+str(person.score)+'\n')
+			else:
+				csv_file.write(str(person.id)+','+str(person.issue)+'\n')
+
+def outputIssues(filename,people):
+	with open(filename, "wb") as csv_file:
+		csv_file.write('Person ID, Issue \n')
+		for person in people:
+			if person.issue is not None:
+				csv_file.write(str(person.id)+','+str(person.issue)+'\n')
+
+# def requestURL(url,payload):
+# 	r = requests.get(url, params=payload)
+# 	time.sleep(5)
+# 	return r
+
+# def loadPickUpdate(filename):
+
+def loadPick(filename):
+	with open(filename, 'rb') as handle:
+		b = pickle.load(handle)
+	return b
+
+def savePick(filename,data):
+	with open(filename, 'wb') as handle:
+		pickle.dump(data,handle)
+
+def parseHoriscope(people,saveFile):
+	rawData = {}
+	for person in people:
+		issue = person.identifyIssues()
+		if issue is not None:
+			print 'SKIPPING person '+ person.id + ' error with ' + issue 
+		else:
+			print 'parsing person '+ person.id
+			if person.resp is None:
+				print 'Posting Request for person '+ person.id
+				person.makePayload()
+				rawData[person.id] = person.requestURL()
+			print 'parsing person '+ person.id
+			person.horiscope = person.parsePage()
+			# person.horiscope.printPositions()
+		if saveFile is not None:
+			savePick('raw_'+saveFile,rawData)
+			savePick(saveFile,people)
+
+def parseHoriscopeRaw(people,saveFile):
+	rawData = loadPick('raw_'+saveFile)
+	for person in people:
+		issue = person.identifyIssues()
+		if issue is not None:
+			print 'SKIPPING person '+ person.id + ' error with ' + issue 
+		else:
+			print 'parsing person '+ person.id
+			if person.resp is None:
+				print 'Posting Request for person '+ person.id
+				person.makePayload()
+				person.resp = rawData[person.id]
+			print 'parsing person '+ person.id
+			person.horiscope = person.parsePage()
+			# person.horiscope.printPositions()
+		if saveFile is not None:
+			savePick(saveFile,people)
+
+# def testMain():
+# 	pickFile = 'outData.pick'
+# 	restartDataFile = 0
+# 	if(restartDataFile):
+# 		# people = makePeople('individuals.csv')
+# 		# savePick(pickFile,people)
+# 		people = loadPick(pickFile)
+# 		parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
+# 		parseHoriscope(people,parseSaveFile)
+# 	else:
+# 		people = loadPick('outData_collect.pick')
+# 	comp = syn.compatibility()
+# 	comp.parseCompatRules('compatibilityRules.csv')
+# 	for person in people:
+# 		if person.issue is None:
+# 			person.score = comp.calcCompatibility(person.horiscope)
+# 			if person.score is None:
+# 				person.issue = 'None Planet Locations'
+# 			else:
+# 				print person.id,person.score
+# 	pdb.set_trace()
+
+def _main():
+	pickFile = 'outData.pick'
+	if not os.path.exists(pickFile):
+		people = makePeople('individuals.csv')
+		savePick(pickFile,people)
+	else:
+		people = loadPick(pickFile)
+	parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
+	if not os.path.exists(parseSaveFile):
+		if os.path.exists('raw_'+parseSaveFile):
+			parseHoriscopeRaw(people,parseSaveFile)
+		else:
+			parseHoriscope(people,parseSaveFile)
+	else:
+		people = loadPick(parseSaveFile)
+	# if not os.path.exists('fullResults.pick'):
+	if not os.path.exists('fullResults.pick'):
+		comp = syn.compatibility()
+		comp.parseCompatRules('compatibilityRules.csv')
+		for person in people:
+			print person.id
+			person.score = None
+			if person.issue is None:
+				person.score = comp.calcCompatibility(person.horiscope)
+				if person.score is None:
+					person.issue = 'None Planet Locations'
+					# pdb.set_trace()
+					f = open('issues/'+str(person.id)+'.html','w')
+					f.write(person.resp.content)
+					f.close()
+		savePick('fullResults.pick',people)
+	else:
+		people = loadPick('fullResults.pick')
+	outputPeople('fullResult.csv',people)
+	outputScores('scores.csv',people)
+	outputIssues('issues.csv',people)
+
+if __name__ == "__main__":
+	_main()