1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 """
23 bundles of files used to implement caching over the network
24 """
25
26 import StringIO
27 import errno
28 import os
29 import sys
30 import tempfile
31 import zipfile
32
33 from flumotion.common import errors, dag, python
34 from flumotion.common.python import makedirs
35
36 __all__ = ['Bundle', 'Bundler', 'Unbundler', 'BundlerBasket']
37 __version__ = "$Rev: 7990 $"
38
39
42
43
45
46
47
48 try:
49 return os.rename(source, dest)
50 except WindowsError, e:
51 import winerror
52 if e.errno == winerror.ERROR_ALREADY_EXISTS:
53 os.unlink(source)
54
55
56 if sys.platform == 'win32':
57 rename = _win32Rename
58
59
61 """
62 I represent one file as managed by a bundler.
63 """
64
65 - def __init__(self, source, destination):
66 self.source = source
67 self.destination = destination
68 self._last_md5sum = None
69 self._last_timestamp = None
70 self.zipped = False
71
73 """
74 Calculate the md5sum of the given file.
75
76 @returns: the md5 sum a 32 character string of hex characters.
77 """
78 data = open(self.source, "r").read()
79 return python.md5(data).hexdigest()
80
82 """
83 @returns: the last modified timestamp for the file.
84 """
85 return os.path.getmtime(self.source)
86
88 """
89 Check if the file has changed since it was last checked.
90
91 @rtype: boolean
92 """
93
94
95
96
97 if not self.zipped:
98 return True
99
100 timestamp = self.timestamp()
101
102
103 if self._last_timestamp and timestamp <= self._last_timestamp:
104 return False
105 self._last_timestamp = timestamp
106
107
108 md5sum = self.md5sum()
109 if self._last_md5sum != md5sum:
110 self._last_md5sum = md5sum
111 return True
112
113 return False
114
115 - def pack(self, zip):
116 self._last_timestamp = self.timestamp()
117 self._last_md5sum = self.md5sum()
118 zip.write(self.source, self.destination)
119 self.zipped = True
120
121
123 """
124 I am a bundle of files, represented by a zip file and md5sum.
125 """
126
131
133 """
134 Set the bundle to the given data representation of the zip file.
135 """
136 self.zip = zip
137 self.md5sum = python.md5(self.zip).hexdigest()
138
140 """
141 Get the bundle's zip data.
142 """
143 return self.zip
144
145
147 """
148 I unbundle bundles by unpacking them in the given directory
149 under directories with the bundle's md5sum.
150 """
151
154
156 """
157 Return the full path where a bundle with the given name and md5sum
158 would be unbundled to.
159 """
160 return os.path.join(self._undir, name, md5sum)
161
167
169 """
170 Unbundle the given bundle.
171
172 @type bundle: L{flumotion.common.bundle.Bundle}
173
174 @rtype: string
175 @returns: the full path to the directory where it was unpacked
176 """
177 directory = self.unbundlePath(bundle)
178
179 filelike = StringIO.StringIO(bundle.getZip())
180 zipFile = zipfile.ZipFile(filelike, "r")
181 zipFile.testzip()
182
183 filepaths = zipFile.namelist()
184 for filepath in filepaths:
185 path = os.path.join(directory, filepath)
186 parent = os.path.split(path)[0]
187 try:
188 makedirs(parent)
189 except OSError, err:
190
191 if err.errno != errno.EEXIST or not os.path.isdir(parent):
192 raise
193 data = zipFile.read(filepath)
194
195
196 fd, tempname = tempfile.mkstemp(dir=parent)
197 handle = os.fdopen(fd, 'wb')
198 handle.write(data)
199 handle.close()
200 rename(tempname, path)
201 return directory
202
203
205 """
206 I bundle files into a bundle so they can be cached remotely easily.
207 """
208
210 """
211 Create a new bundle.
212 """
213 self._bundledFiles = {}
214 self.name = name
215 self._bundle = Bundle(name)
216
217 - def add(self, source, destination = None):
218 """
219 Add files to the bundle.
220
221 @param source: the path to the file to add to the bundle.
222 @param destination: a relative path to store this file in the bundle.
223 If unspecified, this will be stored in the top level.
224
225 @returns: the path the file got stored as
226 """
227 if destination == None:
228 destination = os.path.split(source)[1]
229 self._bundledFiles[source] = BundledFile(source, destination)
230 return destination
231
233 """
234 Bundle the files registered with the bundler.
235
236 @rtype: L{flumotion.common.bundle.Bundle}
237 """
238
239
240 if not self._bundle.getZip():
241 self._bundle.setZip(self._buildzip())
242 return self._bundle
243
244 update = False
245 for bundledFile in self._bundledFiles.values():
246 if bundledFile.hasChanged():
247 update = True
248 break
249
250 if update:
251 self._bundle.setZip(self._buildzip())
252
253 return self._bundle
254
255
256
257
259 filelike = StringIO.StringIO()
260 zipFile = zipfile.ZipFile(filelike, "w")
261 for bundledFile in self._bundledFiles.values():
262 bundledFile.pack(zipFile)
263 zipFile.close()
264 data = filelike.getvalue()
265 filelike.close()
266 return data
267
268
270 """
271 I manage bundlers that are registered through me.
272 """
273
275 """
276 Create a new bundler basket.
277 """
278 self._bundlers = {}
279
280 self._files = {}
281 self._imports = {}
282
283 self._graph = dag.DAG()
284
285 - def add(self, bundleName, source, destination=None):
286 """
287 Add files to the bundler basket for the given bundle.
288
289 @param bundleName: the name of the bundle this file is a part of
290 @param source: the path to the file to add to the bundle
291 @param destination: a relative path to store this file in the bundle.
292 If unspecified, this will be stored in the top level
293 """
294
295 if not bundleName in self._bundlers:
296 bundler = Bundler(bundleName)
297 self._bundlers[bundleName] = bundler
298 else:
299 bundler = self._bundlers[bundleName]
300
301
302 location = bundler.add(source, destination)
303 if location in self._files:
304 raise Exception("Cannot add %s to bundle %s, already in %s" % (
305 location, bundleName, self._files[location]))
306 self._files[location] = bundleName
307
308
309 package = None
310 if location.endswith('.py'):
311 package = location[:-3]
312 elif location.endswith('.pyc'):
313 package = location[:-4]
314
315 if package:
316 if package.endswith('__init__'):
317 package = os.path.split(package)[0]
318
319 package = ".".join(package.split('/'))
320 if package in self._imports:
321 raise Exception("Bundler %s already has import %s" % (
322 bundleName, package))
323 self._imports[package] = bundleName
324
325 - def depend(self, depender, *dependencies):
326 """
327 Make the given bundle depend on the other given bundles.
328
329 @type depender: string
330 @type dependencies: list of strings
331 """
332
333 if not self._graph.hasNode(depender):
334 self._graph.addNode(depender)
335 for dep in dependencies:
336 if not self._graph.hasNode(dep):
337 self._graph.addNode(dep)
338 self._graph.addEdge(depender, dep)
339
341 """
342 Return names of all the dependencies of this bundle, including this
343 bundle itself.
344 The dependencies are returned in a correct depending order.
345 """
346 if not bundlerName in self._bundlers:
347 raise errors.NoBundleError('Unknown bundle %s' % bundlerName)
348 elif not self._graph.hasNode(bundlerName):
349 return [bundlerName]
350 else:
351 return [bundlerName] + self._graph.getOffspring(bundlerName)
352
354 """
355 Return the bundle by name, or None if not found.
356 """
357 if bundlerName in self._bundlers:
358 return self._bundlers[bundlerName]
359 return None
360
362 """
363 Return the bundler name by import statement, or None if not found.
364 """
365 if importString in self._imports:
366 return self._imports[importString]
367 return None
368
370 """
371 Return the bundler name by filename, or None if not found.
372 """
373 if filename in self._files:
374 return self._files[filename]
375 return None
376
378 """
379 Get all bundler names.
380
381 @rtype: list of str
382 @returns: a list of all bundler names in this basket.
383 """
384 return self._bundlers.keys()
385
386
388 """
389 I am a bundler, with the extension that I can also bundle other
390 bundlers.
391
392 The effect is that when you call bundle() on a me, you get one
393 bundle with a union of all subbundlers' files, in addition to any
394 loose files that you added to me.
395 """
396
397 - def __init__(self, name='merged-bundle'):
400
402 """Add to me all of the files managed by another bundler.
403
404 @param bundler: The bundler whose files you want in this
405 bundler.
406 @type bundler: L{Bundler}
407 """
408 if bundler.name not in self._subbundlers:
409 self._subbundlers[bundler.name] = bundler
410 for bfile in bundler._files.values():
411 self.add(bfile.source, bfile.destination)
412
414 """
415 @returns: A list of all of the bundlers that have been added to
416 me.
417 """
418 return self._subbundlers.values()
419