1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 __revision__ = "src/engine/SCons/CacheDir.py 72ae09dc35ac2626f8ff711d8c4b30b6138e08e3 2019-08-08 14:50:06 bdeegan"
25
26 __doc__ = """
27 CacheDir support
28 """
29
30 import hashlib
31 import json
32 import os
33 import stat
34 import sys
35
36 import SCons
37 import SCons.Action
38 import SCons.Warnings
39 from SCons.Util import PY3
40
41 cache_enabled = True
42 cache_debug = False
43 cache_force = False
44 cache_show = False
45 cache_readonly = False
70
79
80 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString)
81
82 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
85 if cache_readonly:
86 return
87
88 t = target[0]
89 if t.nocache:
90 return
91 fs = t.fs
92 cd = env.get_CacheDir()
93 cachedir, cachefile = cd.cachepath(t)
94 if fs.exists(cachefile):
95
96
97
98
99
100
101
102 cd.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile)
103 return
104
105 cd.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile)
106
107 tempfile = cachefile+'.tmp'+str(os.getpid())
108 errfmt = "Unable to copy %s to cache. Cache file is %s"
109
110 if not fs.isdir(cachedir):
111 try:
112 fs.makedirs(cachedir)
113 except EnvironmentError:
114
115
116 if not fs.isdir(cachedir):
117 msg = errfmt % (str(target), cachefile)
118 raise SCons.Errors.SConsEnvironmentError(msg)
119
120 try:
121 if fs.islink(t.get_internal_path()):
122 fs.symlink(fs.readlink(t.get_internal_path()), tempfile)
123 else:
124 fs.copy2(t.get_internal_path(), tempfile)
125 fs.rename(tempfile, cachefile)
126 st = fs.stat(t.get_internal_path())
127 fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE)
128 except EnvironmentError:
129
130
131
132
133
134 msg = errfmt % (str(target), cachefile)
135 SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
136
137 CachePush = SCons.Action.Action(CachePushFunc, None)
138
139
140
141 warned = dict()
144
146 """
147 Initialize a CacheDir object.
148
149 The cache configuration is stored in the object. It
150 is read from the config file in the supplied path if
151 one exists, if not the config file is created and
152 the default config is written, as well as saved in the object.
153 """
154 self.requests = 0
155 self.hits = 0
156 self.path = path
157 self.current_cache_debug = None
158 self.debugFP = None
159 self.config = dict()
160 if path is None:
161 return
162
163 if PY3:
164 self._readconfig3(path)
165 else:
166 self._readconfig2(path)
167
168
170 """
171 Python3 version of reading the cache config.
172
173 If directory or config file do not exist, create. Take advantage
174 of Py3 capability in os.makedirs() and in file open(): just try
175 the operation and handle failure appropriately.
176
177 Omit the check for old cache format, assume that's old enough
178 there will be none of those left to worry about.
179
180 :param path: path to the cache directory
181 """
182 config_file = os.path.join(path, 'config')
183 try:
184 os.makedirs(path, exist_ok=True)
185 except FileExistsError:
186 pass
187 except OSError:
188 msg = "Failed to create cache directory " + path
189 raise SCons.Errors.SConsEnvironmentError(msg)
190
191 try:
192 with open(config_file, 'x') as config:
193 self.config['prefix_len'] = 2
194 try:
195 json.dump(self.config, config)
196 except Exception:
197 msg = "Failed to write cache configuration for " + path
198 raise SCons.Errors.SConsEnvironmentError(msg)
199 except FileExistsError:
200 try:
201 with open(config_file) as config:
202 self.config = json.load(config)
203 except ValueError:
204 msg = "Failed to read cache configuration for " + path
205 raise SCons.Errors.SConsEnvironmentError(msg)
206
207
209 """
210 Python2 version of reading cache config.
211
212 See if there is a config file in the cache directory. If there is,
213 use it. If there isn't, and the directory exists and isn't empty,
214 produce a warning. If the directory does not exist or is empty,
215 write a config file.
216
217 :param path: path to the cache directory
218 """
219 config_file = os.path.join(path, 'config')
220 if not os.path.exists(config_file):
221
222
223
224
225
226
227
228
229 if os.path.isdir(path) and any(f != "config" for f in os.listdir(path)):
230 self.config['prefix_len'] = 1
231
232
233 global warned
234 if self.path not in warned:
235 msg = "Please upgrade your cache by running " +\
236 "scons-configure-cache.py " + self.path
237 SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg)
238 warned[self.path] = True
239 else:
240 if not os.path.isdir(path):
241 try:
242 os.makedirs(path)
243 except OSError:
244
245
246 msg = "Failed to create cache directory " + path
247 raise SCons.Errors.SConsEnvironmentError(msg)
248
249 self.config['prefix_len'] = 2
250 if not os.path.exists(config_file):
251 try:
252 with open(config_file, 'w') as config:
253 json.dump(self.config, config)
254 except Exception:
255 msg = "Failed to write cache configuration for " + path
256 raise SCons.Errors.SConsEnvironmentError(msg)
257 else:
258 try:
259 with open(config_file) as config:
260 self.config = json.load(config)
261 except ValueError:
262 msg = "Failed to read cache configuration for " + path
263 raise SCons.Errors.SConsEnvironmentError(msg)
264
265
267 if cache_debug != self.current_cache_debug:
268 if cache_debug == '-':
269 self.debugFP = sys.stdout
270 elif cache_debug:
271 self.debugFP = open(cache_debug, 'w')
272 else:
273 self.debugFP = None
274 self.current_cache_debug = cache_debug
275 if self.debugFP:
276 self.debugFP.write(fmt % (target, os.path.split(cachefile)[1]))
277 self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" %
278 (self.requests, self.hits, self.misses, self.hit_ratio))
279
280 @property
282 return (100.0 * self.hits / self.requests if self.requests > 0 else 100)
283
284 @property
286 return self.requests - self.hits
287
290
293
295 """
296 """
297 if not self.is_enabled():
298 return None, None
299
300 sig = node.get_cachedir_bsig()
301
302 subdir = sig[:self.config['prefix_len']].upper()
303
304 dir = os.path.join(self.path, subdir)
305 return dir, os.path.join(dir, sig)
306
308 """
309 This method is called from multiple threads in a parallel build,
310 so only do thread safe stuff here. Do thread unsafe stuff in
311 built().
312
313 Note that there's a special trick here with the execute flag
314 (one that's not normally done for other actions). Basically
315 if the user requested a no_exec (-n) build, then
316 SCons.Action.execute_actions is set to 0 and when any action
317 is called, it does its showing but then just returns zero
318 instead of actually calling the action execution operation.
319 The problem for caching is that if the file does NOT exist in
320 cache then the CacheRetrieveString won't return anything to
321 show for the task, but the Action.__call__ won't call
322 CacheRetrieveFunc; instead it just returns zero, which makes
323 the code below think that the file *was* successfully
324 retrieved from the cache, therefore it doesn't do any
325 subsequent building. However, the CacheRetrieveString didn't
326 print anything because it didn't actually exist in the cache,
327 and no more build actions will be performed, so the user just
328 sees nothing. The fix is to tell Action.__call__ to always
329 execute the CacheRetrieveFunc and then have the latter
330 explicitly check SCons.Action.execute_actions itself.
331 """
332 if not self.is_enabled():
333 return False
334
335 env = node.get_build_env()
336 if cache_show:
337 if CacheRetrieveSilent(node, [], env, execute=1) == 0:
338 node.build(presub=0, execute=0)
339 return True
340 else:
341 if CacheRetrieve(node, [], env, execute=1) == 0:
342 return True
343
344 return False
345
346 - def push(self, node):
350
354
355
356
357
358
359
360