|
mi@0
|
1 #!/usr/bin/env python
|
|
mi@0
|
2 # encoding: utf-8
|
|
mi@0
|
3 """
|
|
mi@0
|
4 ComputationCache.py
|
|
mi@0
|
5
|
|
mi@0
|
6 Created by George Fazekas on 2014-09-21.
|
|
mi@0
|
7 Copyright (c) 2014 . All rights reserved.
|
|
mi@0
|
8
|
|
mi@0
|
9 These methods can be used to decorate any function with a caching mechanism to avoid repeating long computations during experimentation.
|
|
mi@0
|
10
|
|
mi@0
|
11 Example:
|
|
mi@0
|
12
|
|
mi@0
|
13 m = Meta()
|
|
mi@0
|
14 m.cache = True
|
|
mi@0
|
15 m.cache_file_base = "filname-with-unique-parameters.txt"
|
|
mi@0
|
16
|
|
mi@0
|
17 @with_cache(meta)
|
|
mi@0
|
18 def some_heavy_function():
|
|
mi@0
|
19 ... lots of computation here...
|
|
mi@0
|
20 return numpy_array
|
|
mi@0
|
21
|
|
mi@0
|
22
|
|
mi@0
|
23 Once decorated, some_heavy_function() will only execute once, then the results will be loaded from a file as long as the parametrs
|
|
mi@0
|
24 encoded in cache_file_base don't change. The parameters and function name are appended to the cache file name so the decorator
|
|
mi@0
|
25 is safe to use for multiple computationally intense functions.
|
|
mi@0
|
26
|
|
mi@0
|
27 The decorator assumes the heavy function returns a singke numpy array or matrix.
|
|
mi@0
|
28
|
|
mi@0
|
29 """
|
|
mi@0
|
30
|
|
mi@0
|
31 import cPickle
|
|
mi@0
|
32 import numpy as np
|
|
mi@0
|
33 from os.path import join, isdir, dirname
|
|
mi@0
|
34
|
|
mi@0
|
35 class Meta(object):
|
|
mi@0
|
36 __slots__ = ["cache","cache_file_base","cache_location"]
|
|
mi@0
|
37
|
|
mi@0
|
38
|
|
mi@0
|
39 def makedir(output_folder) :
|
|
mi@0
|
40 '''Create a directory tree and set privileges to allow acces to multiple users.'''
|
|
mi@0
|
41 if output_folder and not isdir(output_folder) :
|
|
mi@0
|
42 try:
|
|
mi@0
|
43 from os import makedirs
|
|
mi@0
|
44 makedirs(output_folder,0o777)
|
|
mi@0
|
45 except :
|
|
mi@0
|
46 print "Failed to create directory: %s" %output_folder
|
|
mi@0
|
47 import sys
|
|
mi@0
|
48 sys.exit(-1)
|
|
mi@0
|
49 pass
|
|
mi@0
|
50
|
|
mi@0
|
51
|
|
mi@0
|
52 '''Generic decorator that caches function execution results.'''
|
|
mi@0
|
53 def with_cache(meta):
|
|
mi@0
|
54 def wrap(func):
|
|
mi@0
|
55 def file_cache(*args, **kwargs):
|
|
mi@0
|
56 if meta.cache :
|
|
mi@0
|
57 file = "Cache-" + meta.cache_file_base + "-f_%s.txt" %func.__name__
|
|
mi@0
|
58 file = join(getattr(meta,"cache_location",""),file)
|
|
mi@0
|
59 # print func, meta.cache, file
|
|
mi@0
|
60 try :
|
|
mi@0
|
61 print "Loading data from file <%s>" %file
|
|
mi@0
|
62 return np.loadtxt(file)
|
|
mi@0
|
63 except :
|
|
mi@0
|
64 print "Loading from <%s> failed. Computing new results." %file
|
|
mi@0
|
65 makedir(dirname(file))
|
|
mi@0
|
66 result = func(*args, **kwargs)
|
|
mi@0
|
67 np.savetxt(file,result)
|
|
mi@0
|
68 return result
|
|
mi@0
|
69 else :
|
|
mi@0
|
70 return func(*args, **kwargs)
|
|
mi@0
|
71 return file_cache
|
|
mi@0
|
72 return wrap
|
|
mi@0
|
73
|
|
mi@0
|
74 def with_pickle_dump(meta):
|
|
mi@0
|
75 def wrap(func):
|
|
mi@0
|
76 def file_cache(*args, **kwargs):
|
|
mi@0
|
77 if meta.cache :
|
|
mi@0
|
78 file = "Cache-" + meta.cache_file_base + "-f_%s-pickle.txt" %func.__name__
|
|
mi@0
|
79 file = join(getattr(meta,"cache_location",""),file)
|
|
mi@0
|
80 file = file.replace(" ","-")
|
|
mi@0
|
81 # print func, meta.cache, file
|
|
mi@0
|
82 try :
|
|
mi@0
|
83 print "Loading data from file <%s>" %file
|
|
mi@0
|
84 with open(file, 'r') as fh:
|
|
mi@0
|
85 return cPickle.load(fh)
|
|
mi@0
|
86 except :
|
|
mi@0
|
87 print "Loading from <%s> failed. Computing new results." %file
|
|
mi@0
|
88 makedir(dirname(file))
|
|
mi@0
|
89 result = func(*args, **kwargs)
|
|
mi@0
|
90 with open(file, 'w') as f:
|
|
mi@0
|
91 f.write(cPickle.dumps(result))
|
|
mi@0
|
92 return result
|
|
mi@0
|
93 else :
|
|
mi@0
|
94 return func(*args, **kwargs)
|
|
mi@0
|
95 return file_cache
|
|
mi@0
|
96 return wrap
|
|
mi@0
|
97
|
|
mi@0
|
98
|
