annotate V4/runme.py @ 15:50a95089414d

updating to allow for all aspects to be calculated, and remove default value passing for people.

#!/usr/bin/env python


import dParse as dp
import synastry as syn
import requests
import time
import csv
import pdb
import os
import pickle
import sys
from bs4 import BeautifulSoup

def parsePage(resp):
	gotLocation = 0
	horiscope = syn.planetPositions()
	soup = BeautifulSoup(resp.content, 'lxml')
	tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'})
	for cell in tcCell:
		if "Planets in partner's house" in cell.get_text():
			gotLocation = 1
		divList = cell.find_all('div')
		for i in range(len(divList)):
			planetName = divList[i].getText().lower().strip().replace(':','').split(' ')[0]
			if planetName in syn.planetPositions.planetNames:
				if gotLocation and not '/' in planetName:
					horiscope.planets[planetName].setHouse(divList[i+2].getText(),divList[i+4].getText())
				else:
					horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText())
	return horiscope

def makePeople(filename):
	stream = csv.DictReader(open(filename,'rb'))
	dictList = []
	people = []
	for line in stream:
		thisPerson = syn.Person(dp.regulateData(line))
		people.append(thisPerson)
		# pdb.set_trace()
	return people

def requestURL(url,payload):
	r = requests.get(url, params=payload)
	time.sleep(5)
	return r

def loadPick(filename):
	with open(filename, 'rb') as handle:
		b = pickle.load(handle)
	return b

def savePick(filename,data):
	with open(filename, 'wb') as handle:
		pickle.dump(data,handle)

def parseHoriscope(people,saveFile):
	horiscopeList = []
	for person in people:
		issue = person.identifyIssues()
		if issue is not None:
			print 'SKIPPING person '+ person.id + ' error with ' + issue 
		else:
			print 'parsing person '+ person.id
			person.makePayload()
			person.resp = requestURL(person.url,person.payload)
			person.horiscope = parsePage(person.resp)
			# person.horiscope.printPositions()
		if saveFile is not None:
				savePick(saveFile,people)

def testMain():
	restartDataFile = 1
	if(restartDataFile):
		pickFile = 'outData.pick'
		# people = makePeople('individuals.csv')
		# savePick(pickFile,people)
		people = loadPick(pickFile)
		parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
		parseHoriscope(people,parseSaveFile)
	else:
		people = loadPick('onlineDatacollect.pick')
	for p in people:
		if p.horiscope is None:
			print p.id	
		else:
			p.horiscope.calcAllAspects()

	# horiscopeData = presentResults(parseSaveFile)
	# comRules = comp.parseCompatDef('compatibilityRules.csv')
	# applyCompatScore(horiscopeData,rules)

def _main():
	pickFile = 'outData.pick'
	# people = dict()
	if not os.path.exists(pickFile):
		print 'reParse file'
		people = makePeople('individuals.csv')
		savePick(pickFile,people)
	else:
		print 'read in ' + pickFile
		people = loadPick(pickFile)
	parseSaveFile = pickFile.split('.')[0]+'_collect.pick'
	parseHoriscope(people,parseSaveFile)
	horiscopeData = presentResults(parseSaveFile)
	comRules = comp.parseCompatDef('compatibilityRules.csv')
	applyCompatScore(horiscopeData,rules)

if __name__ == "__main__":
	testMain()