Package SCons :: Module CacheDir
[hide private]
[frames] | no frames]

Source Code for Module SCons.CacheDir

  1  # 
  2  # Copyright (c) 2001 - 2019 The SCons Foundation 
  3  # 
  4  # Permission is hereby granted, free of charge, to any person obtaining 
  5  # a copy of this software and associated documentation files (the 
  6  # "Software"), to deal in the Software without restriction, including 
  7  # without limitation the rights to use, copy, modify, merge, publish, 
  8  # distribute, sublicense, and/or sell copies of the Software, and to 
  9  # permit persons to whom the Software is furnished to do so, subject to 
 10  # the following conditions: 
 11  # 
 12  # The above copyright notice and this permission notice shall be included 
 13  # in all copies or substantial portions of the Software. 
 14  # 
 15  # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 
 16  # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 
 17  # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 
 18  # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 
 19  # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 
 20  # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 
 21  # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 22  # 
 23   
 24  __revision__ = "src/engine/SCons/CacheDir.py 72ae09dc35ac2626f8ff711d8c4b30b6138e08e3 2019-08-08 14:50:06 bdeegan" 
 25   
 26  __doc__ = """ 
 27  CacheDir support 
 28  """ 
 29   
 30  import hashlib 
 31  import json 
 32  import os 
 33  import stat 
 34  import sys 
 35   
 36  import SCons 
 37  import SCons.Action 
 38  import SCons.Warnings 
 39  from SCons.Util import PY3 
 40   
 41  cache_enabled = True 
 42  cache_debug = False 
 43  cache_force = False 
 44  cache_show = False 
 45  cache_readonly = False 
46 47 -def CacheRetrieveFunc(target, source, env):
48 t = target[0] 49 fs = t.fs 50 cd = env.get_CacheDir() 51 cd.requests += 1 52 cachedir, cachefile = cd.cachepath(t) 53 if not fs.exists(cachefile): 54 cd.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) 55 return 1 56 cd.hits += 1 57 cd.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) 58 if SCons.Action.execute_actions: 59 if fs.islink(cachefile): 60 fs.symlink(fs.readlink(cachefile), t.get_internal_path()) 61 else: 62 env.copy_from_cache(cachefile, t.get_internal_path()) 63 try: 64 os.utime(cachefile, None) 65 except OSError: 66 pass 67 st = fs.stat(cachefile) 68 fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 69 return 0
70
71 -def CacheRetrieveString(target, source, env):
72 t = target[0] 73 fs = t.fs 74 cd = env.get_CacheDir() 75 cachedir, cachefile = cd.cachepath(t) 76 if t.fs.exists(cachefile): 77 return "Retrieved `%s' from cache" % t.get_internal_path() 78 return None
79 80 CacheRetrieve = SCons.Action.Action(CacheRetrieveFunc, CacheRetrieveString) 81 82 CacheRetrieveSilent = SCons.Action.Action(CacheRetrieveFunc, None)
83 84 -def CachePushFunc(target, source, env):
85 if cache_readonly: 86 return 87 88 t = target[0] 89 if t.nocache: 90 return 91 fs = t.fs 92 cd = env.get_CacheDir() 93 cachedir, cachefile = cd.cachepath(t) 94 if fs.exists(cachefile): 95 # Don't bother copying it if it's already there. Note that 96 # usually this "shouldn't happen" because if the file already 97 # existed in cache, we'd have retrieved the file from there, 98 # not built it. This can happen, though, in a race, if some 99 # other person running the same build pushes their copy to 100 # the cache after we decide we need to build it but before our 101 # build completes. 102 cd.CacheDebug('CachePush(%s): %s already exists in cache\n', t, cachefile) 103 return 104 105 cd.CacheDebug('CachePush(%s): pushing to %s\n', t, cachefile) 106 107 tempfile = cachefile+'.tmp'+str(os.getpid()) 108 errfmt = "Unable to copy %s to cache. Cache file is %s" 109 110 if not fs.isdir(cachedir): 111 try: 112 fs.makedirs(cachedir) 113 except EnvironmentError: 114 # We may have received an exception because another process 115 # has beaten us creating the directory. 116 if not fs.isdir(cachedir): 117 msg = errfmt % (str(target), cachefile) 118 raise SCons.Errors.SConsEnvironmentError(msg) 119 120 try: 121 if fs.islink(t.get_internal_path()): 122 fs.symlink(fs.readlink(t.get_internal_path()), tempfile) 123 else: 124 fs.copy2(t.get_internal_path(), tempfile) 125 fs.rename(tempfile, cachefile) 126 st = fs.stat(t.get_internal_path()) 127 fs.chmod(cachefile, stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) 128 except EnvironmentError: 129 # It's possible someone else tried writing the file at the 130 # same time we did, or else that there was some problem like 131 # the CacheDir being on a separate file system that's full. 132 # In any case, inability to push a file to cache doesn't affect 133 # the correctness of the build, so just print a warning. 134 msg = errfmt % (str(target), cachefile) 135 SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
136 137 CachePush = SCons.Action.Action(CachePushFunc, None) 138 139 # Nasty hack to cut down to one warning for each cachedir path that needs 140 # upgrading. 141 warned = dict()
142 143 -class CacheDir(object):
144
145 - def __init__(self, path):
146 """ 147 Initialize a CacheDir object. 148 149 The cache configuration is stored in the object. It 150 is read from the config file in the supplied path if 151 one exists, if not the config file is created and 152 the default config is written, as well as saved in the object. 153 """ 154 self.requests = 0 155 self.hits = 0 156 self.path = path 157 self.current_cache_debug = None 158 self.debugFP = None 159 self.config = dict() 160 if path is None: 161 return 162 163 if PY3: 164 self._readconfig3(path) 165 else: 166 self._readconfig2(path)
167 168
169 - def _readconfig3(self, path):
170 """ 171 Python3 version of reading the cache config. 172 173 If directory or config file do not exist, create. Take advantage 174 of Py3 capability in os.makedirs() and in file open(): just try 175 the operation and handle failure appropriately. 176 177 Omit the check for old cache format, assume that's old enough 178 there will be none of those left to worry about. 179 180 :param path: path to the cache directory 181 """ 182 config_file = os.path.join(path, 'config') 183 try: 184 os.makedirs(path, exist_ok=True) 185 except FileExistsError: 186 pass 187 except OSError: 188 msg = "Failed to create cache directory " + path 189 raise SCons.Errors.SConsEnvironmentError(msg) 190 191 try: 192 with open(config_file, 'x') as config: 193 self.config['prefix_len'] = 2 194 try: 195 json.dump(self.config, config) 196 except Exception: 197 msg = "Failed to write cache configuration for " + path 198 raise SCons.Errors.SConsEnvironmentError(msg) 199 except FileExistsError: 200 try: 201 with open(config_file) as config: 202 self.config = json.load(config) 203 except ValueError: 204 msg = "Failed to read cache configuration for " + path 205 raise SCons.Errors.SConsEnvironmentError(msg)
206 207
208 - def _readconfig2(self, path):
209 """ 210 Python2 version of reading cache config. 211 212 See if there is a config file in the cache directory. If there is, 213 use it. If there isn't, and the directory exists and isn't empty, 214 produce a warning. If the directory does not exist or is empty, 215 write a config file. 216 217 :param path: path to the cache directory 218 """ 219 config_file = os.path.join(path, 'config') 220 if not os.path.exists(config_file): 221 # A note: There is a race hazard here if two processes start and 222 # attempt to create the cache directory at the same time. However, 223 # Python 2.x does not give you the option to do exclusive file 224 # creation (not even the option to error on opening an existing 225 # file for writing...). The ordering of events here is an attempt 226 # to alleviate this, on the basis that it's a pretty unlikely 227 # occurrence (would require two builds with a brand new cache 228 # directory) 229 if os.path.isdir(path) and any(f != "config" for f in os.listdir(path)): 230 self.config['prefix_len'] = 1 231 # When building the project I was testing this on, the warning 232 # was output over 20 times. That seems excessive 233 global warned 234 if self.path not in warned: 235 msg = "Please upgrade your cache by running " +\ 236 "scons-configure-cache.py " + self.path 237 SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) 238 warned[self.path] = True 239 else: 240 if not os.path.isdir(path): 241 try: 242 os.makedirs(path) 243 except OSError: 244 # If someone else is trying to create the directory at 245 # the same time as me, bad things will happen 246 msg = "Failed to create cache directory " + path 247 raise SCons.Errors.SConsEnvironmentError(msg) 248 249 self.config['prefix_len'] = 2 250 if not os.path.exists(config_file): 251 try: 252 with open(config_file, 'w') as config: 253 json.dump(self.config, config) 254 except Exception: 255 msg = "Failed to write cache configuration for " + path 256 raise SCons.Errors.SConsEnvironmentError(msg) 257 else: 258 try: 259 with open(config_file) as config: 260 self.config = json.load(config) 261 except ValueError: 262 msg = "Failed to read cache configuration for " + path 263 raise SCons.Errors.SConsEnvironmentError(msg)
264 265
266 - def CacheDebug(self, fmt, target, cachefile):
267 if cache_debug != self.current_cache_debug: 268 if cache_debug == '-': 269 self.debugFP = sys.stdout 270 elif cache_debug: 271 self.debugFP = open(cache_debug, 'w') 272 else: 273 self.debugFP = None 274 self.current_cache_debug = cache_debug 275 if self.debugFP: 276 self.debugFP.write(fmt % (target, os.path.split(cachefile)[1])) 277 self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" % 278 (self.requests, self.hits, self.misses, self.hit_ratio))
279 280 @property
281 - def hit_ratio(self):
282 return (100.0 * self.hits / self.requests if self.requests > 0 else 100)
283 284 @property
285 - def misses(self):
286 return self.requests - self.hits
287
288 - def is_enabled(self):
289 return cache_enabled and self.path is not None
290
291 - def is_readonly(self):
292 return cache_readonly
293
294 - def cachepath(self, node):
295 """ 296 """ 297 if not self.is_enabled(): 298 return None, None 299 300 sig = node.get_cachedir_bsig() 301 302 subdir = sig[:self.config['prefix_len']].upper() 303 304 dir = os.path.join(self.path, subdir) 305 return dir, os.path.join(dir, sig)
306
307 - def retrieve(self, node):
308 """ 309 This method is called from multiple threads in a parallel build, 310 so only do thread safe stuff here. Do thread unsafe stuff in 311 built(). 312 313 Note that there's a special trick here with the execute flag 314 (one that's not normally done for other actions). Basically 315 if the user requested a no_exec (-n) build, then 316 SCons.Action.execute_actions is set to 0 and when any action 317 is called, it does its showing but then just returns zero 318 instead of actually calling the action execution operation. 319 The problem for caching is that if the file does NOT exist in 320 cache then the CacheRetrieveString won't return anything to 321 show for the task, but the Action.__call__ won't call 322 CacheRetrieveFunc; instead it just returns zero, which makes 323 the code below think that the file *was* successfully 324 retrieved from the cache, therefore it doesn't do any 325 subsequent building. However, the CacheRetrieveString didn't 326 print anything because it didn't actually exist in the cache, 327 and no more build actions will be performed, so the user just 328 sees nothing. The fix is to tell Action.__call__ to always 329 execute the CacheRetrieveFunc and then have the latter 330 explicitly check SCons.Action.execute_actions itself. 331 """ 332 if not self.is_enabled(): 333 return False 334 335 env = node.get_build_env() 336 if cache_show: 337 if CacheRetrieveSilent(node, [], env, execute=1) == 0: 338 node.build(presub=0, execute=0) 339 return True 340 else: 341 if CacheRetrieve(node, [], env, execute=1) == 0: 342 return True 343 344 return False
345
346 - def push(self, node):
347 if self.is_readonly() or not self.is_enabled(): 348 return 349 return CachePush(node, [], node.get_build_env())
350
351 - def push_if_forced(self, node):
352 if cache_force: 353 return self.push(node)
354 355 # Local Variables: 356 # tab-width:4 357 # indent-tabs-mode:nil 358 # End: 359 # vim: set expandtab tabstop=4 shiftwidth=4: 360