# HG changeset patch # User DaveM # Date 1520357138 0 # Node ID ae220e89cb3aa5dcb57c9d32a8021caec47b8452 # Parent 155126861c07eb8770931061afd7af72c481b8d8 fixing parse bugs, and angle calculation bugs diff -r 155126861c07 -r ae220e89cb3a V4/dParse.py --- a/V4/dParse.py Mon Mar 05 14:23:45 2018 +0000 +++ b/V4/dParse.py Tue Mar 06 17:25:38 2018 +0000 @@ -8,8 +8,8 @@ import random import pdb -DEFAULT_TIME_H = None -DEFAULT_TIME_M = None +DEFAULT_TIME = None +# DEFAULT_TIME_M = None DEAULT_LOCATION = None def regulateData(dataDict): @@ -190,7 +190,6 @@ return (d,m,y) def parseTOB(T): - # pdb.set_trace() timeFlag = None T_ = T.replace('.','').lower().strip() if 'am' in T_: @@ -218,14 +217,16 @@ elif int(T) > 100: H = int(T)/100 M = int(T)%100 + else: + return None if timeFlag is not None: if timeFlag == 0: H = H%12 else: H = H%12 + 12 except ValueError: - H = DEFAULT_TIME_H - M = DEFAULT_TIME_M + return None return (H,M) + diff -r 155126861c07 -r ae220e89cb3a V4/runme.py --- a/V4/runme.py Mon Mar 05 14:23:45 2018 +0000 +++ b/V4/runme.py Tue Mar 06 17:25:38 2018 +0000 @@ -3,8 +3,8 @@ import dParse as dp import synastry as syn -import requests -import time +# import requests +# import time import csv import pdb import os @@ -12,23 +12,23 @@ import sys from bs4 import BeautifulSoup -def parsePage(resp): - gotLocation = 0 - horiscope = syn.planetPositions() - soup = BeautifulSoup(resp.content, 'lxml') - tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'}) - for cell in tcCell: - if "Planets in partner's house" in cell.get_text(): - gotLocation = 1 - divList = cell.find_all('div') - for i in range(len(divList)): - planetName = divList[i].getText().lower().strip().replace(':','').split(' ')[0] - if planetName in syn.planetPositions.planetNames: - if gotLocation and not '/' in planetName: - horiscope.planets[planetName].setHouse(divList[i+2].getText(),divList[i+4].getText()) - else: - horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText()) - return horiscope +# def parsePage(resp): +# gotLocation = 0 +# horiscope = syn.planetPositions() +# soup = BeautifulSoup(resp.content, 'lxml') +# tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'}) +# for cell in tcCell: +# if "Planets in partner's house" in cell.get_text(): +# gotLocation = 1 +# divList = cell.find_all('div') +# for i in range(len(divList)): +# planetName = divList[i].getText().lower().strip().replace(':','').split(' ')[0] +# if planetName in syn.planetPositions.planetNames: +# if gotLocation and not '/' in planetName: +# horiscope.planets[planetName].setHouse(divList[i+2].getText(),divList[i+4].getText()) +# else: +# horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+4].getText()) +# return horiscope def makePeople(filename): stream = csv.DictReader(open(filename,'rb')) @@ -40,10 +40,49 @@ # pdb.set_trace() return people -def requestURL(url,payload): - r = requests.get(url, params=payload) - time.sleep(5) - return r +def uniquify(itemList): + keyDict = {} + for item in itemList: + keyDict[item] = 1 + return keyDict.keys() + +def outputPeople(filename,people): + with open(filename, "wb") as csv_file: + dictKeys = [] + for person in people: + if person.issue is None: + person.horiscope.calcAllAspects() + dictKeys += person.horiscope.aspect.keys() + dictKeys = uniquify(dictKeys) + writer = csv.DictWriter(csv_file, ['id']+dictKeys) + writer.writeheader() + for person in people: + if person.issue is None: + tempDict = {'id':person.id} + tempDict.update(person.horiscope.aspect) + # pdb.set_trace() + writer.writerow(tempDict) + +def outputScores(filename,people): + with open(filename, "wb") as csv_file: + csv_file.write('Person ID, Score \n') + for person in people: + if person.issue is None: + csv_file.write(str(person.id)+','+str(person.score)+'\n') + else: + csv_file.write(str(person.id)+','+str(person.issue)+'\n') + +def outputIssues(filename,people): + with open(filename, "wb") as csv_file: + csv_file.write('Person ID, Issue \n') + for person in people: + if person.issue is not None: + csv_file.write(str(person.id)+','+str(person.issue)+'\n') + +# def requestURL(url,payload): +# r = requests.get(url, params=payload) +# time.sleep(5) +# return r def loadPick(filename): with open(filename, 'rb') as handle: @@ -62,8 +101,8 @@ else: print 'parsing person '+ person.id person.makePayload() - person.resp = requestURL(person.url,person.payload) - person.horiscope = parsePage(person.resp) + person.requestURL() + person.horiscope = person.parsePage() # person.horiscope.printPositions() if saveFile is not None: savePick(saveFile,people) @@ -102,15 +141,24 @@ parseHoriscope(people,parseSaveFile) else: people = loadPick(parseSaveFile) - comp = syn.compatibility() - comp.parseCompatRules('compatibilityRules.csv') - for person in people: - person.score = None - if person.issue is None: - person.score = comp.calcCompatibility(person.horiscope) - if person.score is None: - person.issue = 'None Planet Locations' - savePick('fullResults.pick',people) + if not os.path.exists('fullResults.pick'): + comp = syn.compatibility() + comp.parseCompatRules('compatibilityRules.csv') + for person in people: + person.score = None + if person.issue is None: + person.score = comp.calcCompatibility(person.horiscope) + if person.score is None: + person.issue = 'None Planet Locations' + # pdb.set_trace() + f = open('issues/'+str(person.id)+'.html','w') + f.write(person.resp.content) + f.close() + savePick('fullResults.pick',people) + else: + people = loadPick('fullResults.pick') + outputPeople('fullResult.csv',people) + outputScores('scores.csv',people) if __name__ == "__main__": _main() diff -r 155126861c07 -r ae220e89cb3a V4/synastry.py --- a/V4/synastry.py Mon Mar 05 14:23:45 2018 +0000 +++ b/V4/synastry.py Tue Mar 06 17:25:38 2018 +0000 @@ -1,6 +1,10 @@ import random import csv import re +import requests +import time +from bs4 import BeautifulSoup + import pdb class compatibility: @@ -31,15 +35,13 @@ def parseCompatRules(self,filename): comList = self.readFile(filename) - # compat = {} for com in comList: c = com.split(',') - # print c if len(c) == 4: if len(c[1]) > 1: aspectList = [] planetTuple = tuple(sorted((c[0].lower(),c[2].lower()))) - score = c[3] + score = int(c[3]) for subset in c[1].split(' '): aspectList.append(subset) elif len(c[1]) == 0 : @@ -52,8 +54,6 @@ aspectList = int(houseNo[0]) score = int(c[3]) self.addRule(planetTuple,aspectList,score) - # return compat - # pdb.set_trace() def calcCompatibility(self, horiscope): score = 0 @@ -78,6 +78,7 @@ score += r.score return score + class compatibilityRule: @@ -136,10 +137,7 @@ self.tob = self.tob[1:-1].split(',') if type(self.p_tob) is str: self.p_tob = self.p_tob[1:-1].split(',') - # pdb.set_trace() - - # print dataDict['pDOB'] - + self.payload = {'send_calculation':'1', #Req 'muz_narozeni_den':self.dob[0], 'muz_narozeni_mesic':self.dob[1], @@ -188,9 +186,36 @@ 'house_system':'placidus', 'uhel_orbis':'#tabs_redraw'} + def requestURL(self): + self.resp = requests.get(self.url, params=self.payload) + time.sleep(5) + return self.resp + + def parsePage(self): + gotLocation = 0 + self.horiscope = planetPositions() + soup = BeautifulSoup(self.resp.content, 'lxml') + tcCell = soup.find_all('div', attrs={'class':'right-sedy-banner-svetlejsi'}) + for cell in tcCell: + if "Planets in partner's house" in cell.get_text(): + gotLocation = 1 + divList = cell.find_all('div') + for i in range(len(divList)): + planetName = divList[i].getText().lower().strip().replace(':','').split(' ')[0] + if planetName in planetPositions.planetNames: + if gotLocation and not '/' in planetName: + self.horiscope.planets[planetName].setHouse(divList[i+2].getText(),divList[i+4].getText()) + else: + self.horiscope.planets[planetName].setLocation(divList[i+2].getText(),divList[i+1].img.attrs['alt'],0) + self.horiscope.planets[planetName].setLocation(divList[i+4].getText(),divList[i+3].img.attrs['alt'],1) + return self.horiscope + + class planetRelation: noHouseList = ['asc','ic','dsc','mc','asc/mc','sun/moon'] + zodiacAngle = {'aries':0,'taurus':30,'gemini':60,'cancer':90,'leo':120,'virgo':150,'libra':180,'scorpio':210,'sagittarius':240,'capricorn':270,'aquarius':300,'pisces':330} + def __init__(self,planetName): self.name = planetName self.angleA = None @@ -199,15 +224,18 @@ self.houseA = None self.houseB = None - def setLocation(self,A,B): - self.angleA = float('.'.join(A.encode('ascii','replace').split('?'))[:-1]) - self.angleB = float('.'.join(B.encode('ascii','replace').split('?'))[:-1]) - # print self.angleA,self.angleB - + def setLocation(self,value,sign,isB): + signVal = planetRelation.zodiacAngle[sign.lower()] + signVal+= float('.'.join(value.encode('ascii','replace').split('?'))[:-1]) + # print self.name,sign.lower(),planetRelation.zodiacAngle[sign.lower()],value,signVal + if not isB: + self.angleA = signVal + else: + self.angleB = signVal + def setHouse(self,A,B): self.houseA = int(A.encode('ascii','ignore')) self.houseB = int(B.encode('ascii','ignore')) - # print self.houseA,self.houseB def test_random(self): self.angleA = random.random()*360 @@ -216,6 +244,8 @@ class planetPositions: aspectDict = {'conjunction':0,'semi-square':45,'sextile':60,'square':90,'trine':120,'opposite':180} + # zodiacAngle = {'aries':0,'taurus':30,'gemini':60,'cancer':90,'leo':120,'virgo':150,'libra':180,'scorpio':210,'sagittarius':240,'capricorn':270,'aquarius':300,'pisces':330} + # Taken from https://en.wikipedia.org/wiki/Astrological_sign, with reference from https://en.wikipedia.org/wiki/Astrological_symbols#Signs_of_the_zodiac planetNames = ['sun','moon','mercury','venus','mars','jupiter','saturn','uranus','neptune','pluto','node','lilith','chiron','asc','ic','dsc','mc','asc/mc','sun/moon'] def __init__(self): self.planets = {} @@ -228,14 +258,16 @@ self.planets[planet].test_random() def calcAngle(self,componentA,componentB): - self.angle = max(self.planets[componentA].angleA,self.planets[componentB].angleB) - min(self.planets[componentA].angleA,self.planets[componentB].angleB) + self.angle = abs(self.planets[componentA].angleA - self.planets[componentB].angleB) self.angleRange = self.angle-10,self.angle+10 def calcAspect(self,componentA,componentB): self.calcAngle(componentA,componentB) + # print componentA,componentB,self.angle,self.planets[componentA].angleA,self.planets[componentB].angleB for aspect in planetPositions.aspectDict: if self.angleRange[0] < planetPositions.aspectDict[aspect] and self.angleRange[1] > planetPositions.aspectDict[aspect]: - aspectDiff = abs(self.angle - planetPositions.aspectDict[aspect]) + # print aspect#,componentA,componentB,self.angle + aspectDiff = round(abs(self.angle - planetPositions.aspectDict[aspect]),2) self.aspect[componentA,componentB] = (aspect,aspectDiff) return aspect,aspectDiff return None,None diff -r 155126861c07 -r ae220e89cb3a V4/test.py --- a/V4/test.py Mon Mar 05 14:23:45 2018 +0000 +++ b/V4/test.py Tue Mar 06 17:25:38 2018 +0000 @@ -18,9 +18,8 @@ import synastry as syn import pdb -comp = syn.compatibility() -comp.parseCompatRules('compatibilityRules.csv') - -print [r.planet for r in comp.rules] +horiscope = syn.planetPositions() +horiscope.test_random() +horiscope.calcAllAspects() pdb.set_trace() \ No newline at end of file