Package CedarBackup2 :: Module util
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.util

   1  # -*- coding: iso-8859-1 -*- 
   2  # vim: set ft=python ts=3 sw=3 expandtab: 
   3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
   4  # 
   5  #              C E D A R 
   6  #          S O L U T I O N S       "Software done right." 
   7  #           S O F T W A R E 
   8  # 
   9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  10  # 
  11  # Copyright (c) 2004-2008,2010 Kenneth J. Pronovici. 
  12  # All rights reserved. 
  13  # 
  14  # Portions copyright (c) 2001, 2002 Python Software Foundation. 
  15  # All Rights Reserved. 
  16  # 
  17  # This program is free software; you can redistribute it and/or 
  18  # modify it under the terms of the GNU General Public License, 
  19  # Version 2, as published by the Free Software Foundation. 
  20  # 
  21  # This program is distributed in the hope that it will be useful, 
  22  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  23  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
  24  # 
  25  # Copies of the GNU General Public License are available from 
  26  # the Free Software Foundation website, http://www.gnu.org/. 
  27  # 
  28  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  29  # 
  30  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
  31  # Language : Python 2 (>= 2.7) 
  32  # Project  : Cedar Backup, release 2 
  33  # Purpose  : Provides general-purpose utilities. 
  34  # 
  35  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  36   
  37  ######################################################################## 
  38  # Module documentation 
  39  ######################################################################## 
  40   
  41  """ 
  42  Provides general-purpose utilities. 
  43   
  44  @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList, 
  45         RegexList, _Vertex, DirectedGraph, PathResolverSingleton, 
  46         sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine, 
  47         resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice, 
  48         deriveDayOfWeek, isStartOfWeek, buildNormalizedPath, 
  49         ISO_SECTOR_SIZE, BYTES_PER_SECTOR, 
  50         BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE, 
  51         SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY, 
  52         UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS 
  53   
  54  @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes. 
  55  @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector. 
  56  @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB). 
  57  @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB). 
  58  @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB). 
  59  @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB). 
  60  @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB). 
  61  @var SECONDS_PER_MINUTE: Number of seconds per minute. 
  62  @var MINUTES_PER_HOUR: Number of minutes per hour. 
  63  @var HOURS_PER_DAY: Number of hours per day. 
  64  @var SECONDS_PER_DAY: Number of seconds per day. 
  65  @var UNIT_BYTES: Constant representing the byte (B) unit for conversion. 
  66  @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion. 
  67  @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion. 
  68  @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion. 
  69  @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion. 
  70   
  71  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  72  """ 
  73   
  74   
  75  ######################################################################## 
  76  # Imported modules 
  77  ######################################################################## 
  78   
  79  import sys 
  80  import math 
  81  import os 
  82  import re 
  83  import time 
  84  import logging 
  85  import string  # pylint: disable=W0402 
  86  from subprocess import Popen, STDOUT, PIPE 
  87   
  88  try: 
  89     import pwd 
  90     import grp 
  91     _UID_GID_AVAILABLE = True 
  92  except ImportError: 
  93     _UID_GID_AVAILABLE = False 
  94   
  95  from CedarBackup2.release import VERSION, DATE 
  96   
  97   
  98  ######################################################################## 
  99  # Module-wide constants and variables 
 100  ######################################################################## 
 101   
 102  logger = logging.getLogger("CedarBackup2.log.util") 
 103  outputLogger = logging.getLogger("CedarBackup2.output") 
 104   
 105  ISO_SECTOR_SIZE    = 2048.0   # in bytes 
 106  BYTES_PER_SECTOR   = ISO_SECTOR_SIZE 
 107   
 108  BYTES_PER_KBYTE    = 1024.0 
 109  KBYTES_PER_MBYTE   = 1024.0 
 110  MBYTES_PER_GBYTE   = 1024.0 
 111  BYTES_PER_MBYTE    = BYTES_PER_KBYTE * KBYTES_PER_MBYTE 
 112  BYTES_PER_GBYTE    = BYTES_PER_MBYTE * MBYTES_PER_GBYTE 
 113   
 114  SECONDS_PER_MINUTE = 60.0 
 115  MINUTES_PER_HOUR   = 60.0 
 116  HOURS_PER_DAY      = 24.0 
 117  SECONDS_PER_DAY    = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY 
 118   
 119  UNIT_BYTES         = 0 
 120  UNIT_KBYTES        = 1 
 121  UNIT_MBYTES        = 2 
 122  UNIT_GBYTES        = 4 
 123  UNIT_SECTORS       = 3 
 124   
 125  MTAB_FILE          = "/etc/mtab" 
 126   
 127  MOUNT_COMMAND      = [ "mount", ] 
 128  UMOUNT_COMMAND     = [ "umount", ] 
 129   
 130  DEFAULT_LANGUAGE   = "C" 
 131  LANG_VAR           = "LANG" 
 132  LOCALE_VARS        = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE", 
 133                         "LC_CTYPE", "LC_IDENTIFICATION", 
 134                         "LC_MEASUREMENT", "LC_MESSAGES", 
 135                         "LC_MONETARY", "LC_NAME", "LC_NUMERIC", 
 136                         "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ] 
137 138 139 ######################################################################## 140 # UnorderedList class definition 141 ######################################################################## 142 143 -class UnorderedList(list):
144 145 """ 146 Class representing an "unordered list". 147 148 An "unordered list" is a list in which only the contents matter, not the 149 order in which the contents appear in the list. 150 151 For instance, we might be keeping track of set of paths in a list, because 152 it's convenient to have them in that form. However, for comparison 153 purposes, we would only care that the lists contain exactly the same 154 contents, regardless of order. 155 156 I have come up with two reasonable ways of doing this, plus a couple more 157 that would work but would be a pain to implement. My first method is to 158 copy and sort each list, comparing the sorted versions. This will only work 159 if two lists with exactly the same members are guaranteed to sort in exactly 160 the same order. The second way would be to create two Sets and then compare 161 the sets. However, this would lose information about any duplicates in 162 either list. I've decided to go with option #1 for now. I'll modify this 163 code if I run into problems in the future. 164 165 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__}, 166 C{__le__} and C{__lt__} list methods to change the definition of the various 167 comparison operators. In all cases, the comparison is changed to return the 168 result of the original operation I{but instead comparing sorted lists}. 169 This is going to be quite a bit slower than a normal list, so you probably 170 only want to use it on small lists. 171 """ 172
173 - def __eq__(self, other):
174 """ 175 Definition of C{==} operator for this class. 176 @param other: Other object to compare to. 177 @return: True/false depending on whether C{self == other}. 178 """ 179 if other is None: 180 return False 181 selfSorted = self[:] 182 otherSorted = other[:] 183 selfSorted.sort() 184 otherSorted.sort() 185 return selfSorted.__eq__(otherSorted)
186
187 - def __ne__(self, other):
188 """ 189 Definition of C{!=} operator for this class. 190 @param other: Other object to compare to. 191 @return: True/false depending on whether C{self != other}. 192 """ 193 if other is None: 194 return True 195 selfSorted = self[:] 196 otherSorted = other[:] 197 selfSorted.sort() 198 otherSorted.sort() 199 return selfSorted.__ne__(otherSorted)
200
201 - def __ge__(self, other):
202 """ 203 Definition of S{>=} operator for this class. 204 @param other: Other object to compare to. 205 @return: True/false depending on whether C{self >= other}. 206 """ 207 if other is None: 208 return True 209 selfSorted = self[:] 210 otherSorted = other[:] 211 selfSorted.sort() 212 otherSorted.sort() 213 return selfSorted.__ge__(otherSorted)
214
215 - def __gt__(self, other):
216 """ 217 Definition of C{>} operator for this class. 218 @param other: Other object to compare to. 219 @return: True/false depending on whether C{self > other}. 220 """ 221 if other is None: 222 return True 223 selfSorted = self[:] 224 otherSorted = other[:] 225 selfSorted.sort() 226 otherSorted.sort() 227 return selfSorted.__gt__(otherSorted)
228
229 - def __le__(self, other):
230 """ 231 Definition of S{<=} operator for this class. 232 @param other: Other object to compare to. 233 @return: True/false depending on whether C{self <= other}. 234 """ 235 if other is None: 236 return False 237 selfSorted = self[:] 238 otherSorted = other[:] 239 selfSorted.sort() 240 otherSorted.sort() 241 return selfSorted.__le__(otherSorted)
242
243 - def __lt__(self, other):
244 """ 245 Definition of C{<} operator for this class. 246 @param other: Other object to compare to. 247 @return: True/false depending on whether C{self < other}. 248 """ 249 if other is None: 250 return False 251 selfSorted = self[:] 252 otherSorted = other[:] 253 selfSorted.sort() 254 otherSorted.sort() 255 return selfSorted.__lt__(otherSorted)
256
257 258 ######################################################################## 259 # AbsolutePathList class definition 260 ######################################################################## 261 262 -class AbsolutePathList(UnorderedList):
263 264 """ 265 Class representing a list of absolute paths. 266 267 This is an unordered list. 268 269 We override the C{append}, C{insert} and C{extend} methods to ensure that 270 any item added to the list is an absolute path. 271 272 Each item added to the list is encoded using L{encodePath}. If we don't do 273 this, we have problems trying certain operations between strings and unicode 274 objects, particularly for "odd" filenames that can't be encoded in standard 275 ASCII. 276 """ 277
278 - def append(self, item):
279 """ 280 Overrides the standard C{append} method. 281 @raise ValueError: If item is not an absolute path. 282 """ 283 if not os.path.isabs(item): 284 raise ValueError("Not an absolute path: [%s]" % item) 285 list.append(self, encodePath(item))
286
287 - def insert(self, index, item):
288 """ 289 Overrides the standard C{insert} method. 290 @raise ValueError: If item is not an absolute path. 291 """ 292 if not os.path.isabs(item): 293 raise ValueError("Not an absolute path: [%s]" % item) 294 list.insert(self, index, encodePath(item))
295
296 - def extend(self, seq):
297 """ 298 Overrides the standard C{insert} method. 299 @raise ValueError: If any item is not an absolute path. 300 """ 301 for item in seq: 302 if not os.path.isabs(item): 303 raise ValueError("Not an absolute path: [%s]" % item) 304 for item in seq: 305 list.append(self, encodePath(item))
306
307 308 ######################################################################## 309 # ObjectTypeList class definition 310 ######################################################################## 311 312 -class ObjectTypeList(UnorderedList):
313 314 """ 315 Class representing a list containing only objects with a certain type. 316 317 This is an unordered list. 318 319 We override the C{append}, C{insert} and C{extend} methods to ensure that 320 any item added to the list matches the type that is requested. The 321 comparison uses the built-in C{isinstance}, which should allow subclasses of 322 of the requested type to be added to the list as well. 323 324 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a 325 CollectDir object."} if C{objectName} is C{"CollectDir"}. 326 """ 327
328 - def __init__(self, objectType, objectName):
329 """ 330 Initializes a typed list for a particular type. 331 @param objectType: Type that the list elements must match. 332 @param objectName: Short string containing the "name" of the type. 333 """ 334 super(ObjectTypeList, self).__init__() 335 self.objectType = objectType 336 self.objectName = objectName
337
338 - def append(self, item):
339 """ 340 Overrides the standard C{append} method. 341 @raise ValueError: If item does not match requested type. 342 """ 343 if not isinstance(item, self.objectType): 344 raise ValueError("Item must be a %s object." % self.objectName) 345 list.append(self, item)
346
347 - def insert(self, index, item):
348 """ 349 Overrides the standard C{insert} method. 350 @raise ValueError: If item does not match requested type. 351 """ 352 if not isinstance(item, self.objectType): 353 raise ValueError("Item must be a %s object." % self.objectName) 354 list.insert(self, index, item)
355
356 - def extend(self, seq):
357 """ 358 Overrides the standard C{insert} method. 359 @raise ValueError: If item does not match requested type. 360 """ 361 for item in seq: 362 if not isinstance(item, self.objectType): 363 raise ValueError("All items must be %s objects." % self.objectName) 364 list.extend(self, seq)
365
366 367 ######################################################################## 368 # RestrictedContentList class definition 369 ######################################################################## 370 371 -class RestrictedContentList(UnorderedList):
372 373 """ 374 Class representing a list containing only object with certain values. 375 376 This is an unordered list. 377 378 We override the C{append}, C{insert} and C{extend} methods to ensure that 379 any item added to the list is among the valid values. We use a standard 380 comparison, so pretty much anything can be in the list of valid values. 381 382 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be 383 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}. 384 385 @note: This class doesn't make any attempt to trap for nonsensical 386 arguments. All of the values in the values list should be of the same type 387 (i.e. strings). Then, all list operations also need to be of that type 388 (i.e. you should always insert or append just strings). If you mix types -- 389 for instance lists and strings -- you will likely see AttributeError 390 exceptions or other problems. 391 """ 392
393 - def __init__(self, valuesList, valuesDescr, prefix=None):
394 """ 395 Initializes a list restricted to containing certain values. 396 @param valuesList: List of valid values. 397 @param valuesDescr: Short string describing list of values. 398 @param prefix: Prefix to use in error messages (None results in prefix "Item") 399 """ 400 super(RestrictedContentList, self).__init__() 401 self.prefix = "Item" 402 if prefix is not None: self.prefix = prefix 403 self.valuesList = valuesList 404 self.valuesDescr = valuesDescr
405
406 - def append(self, item):
407 """ 408 Overrides the standard C{append} method. 409 @raise ValueError: If item is not in the values list. 410 """ 411 if item not in self.valuesList: 412 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 413 list.append(self, item)
414
415 - def insert(self, index, item):
416 """ 417 Overrides the standard C{insert} method. 418 @raise ValueError: If item is not in the values list. 419 """ 420 if item not in self.valuesList: 421 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 422 list.insert(self, index, item)
423
424 - def extend(self, seq):
425 """ 426 Overrides the standard C{insert} method. 427 @raise ValueError: If item is not in the values list. 428 """ 429 for item in seq: 430 if item not in self.valuesList: 431 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 432 list.extend(self, seq)
433
434 435 ######################################################################## 436 # RegexMatchList class definition 437 ######################################################################## 438 439 -class RegexMatchList(UnorderedList):
440 441 """ 442 Class representing a list containing only strings that match a regular expression. 443 444 If C{emptyAllowed} is passed in as C{False}, then empty strings are 445 explicitly disallowed, even if they happen to match the regular expression. 446 (C{None} values are always disallowed, since string operations are not 447 permitted on C{None}.) 448 449 This is an unordered list. 450 451 We override the C{append}, C{insert} and C{extend} methods to ensure that 452 any item added to the list matches the indicated regular expression. 453 454 @note: If you try to put values that are not strings into the list, you will 455 likely get either TypeError or AttributeError exceptions as a result. 456 """ 457
458 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
459 """ 460 Initializes a list restricted to containing certain values. 461 @param valuesRegex: Regular expression that must be matched, as a string 462 @param emptyAllowed: Indicates whether empty or None values are allowed. 463 @param prefix: Prefix to use in error messages (None results in prefix "Item") 464 """ 465 super(RegexMatchList, self).__init__() 466 self.prefix = "Item" 467 if prefix is not None: self.prefix = prefix 468 self.valuesRegex = valuesRegex 469 self.emptyAllowed = emptyAllowed 470 self.pattern = re.compile(self.valuesRegex)
471
472 - def append(self, item):
473 """ 474 Overrides the standard C{append} method. 475 @raise ValueError: If item is None 476 @raise ValueError: If item is empty and empty values are not allowed 477 @raise ValueError: If item does not match the configured regular expression 478 """ 479 if item is None or (not self.emptyAllowed and item == ""): 480 raise ValueError("%s cannot be empty." % self.prefix) 481 if not self.pattern.search(item): 482 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 483 list.append(self, item)
484
485 - def insert(self, index, item):
486 """ 487 Overrides the standard C{insert} method. 488 @raise ValueError: If item is None 489 @raise ValueError: If item is empty and empty values are not allowed 490 @raise ValueError: If item does not match the configured regular expression 491 """ 492 if item is None or (not self.emptyAllowed and item == ""): 493 raise ValueError("%s cannot be empty." % self.prefix) 494 if not self.pattern.search(item): 495 raise ValueError("%s is not valid [%s]" % (self.prefix, item)) 496 list.insert(self, index, item)
497
498 - def extend(self, seq):
499 """ 500 Overrides the standard C{insert} method. 501 @raise ValueError: If any item is None 502 @raise ValueError: If any item is empty and empty values are not allowed 503 @raise ValueError: If any item does not match the configured regular expression 504 """ 505 for item in seq: 506 if item is None or (not self.emptyAllowed and item == ""): 507 raise ValueError("%s cannot be empty." % self.prefix) 508 if not self.pattern.search(item): 509 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 510 list.extend(self, seq)
511
512 513 ######################################################################## 514 # RegexList class definition 515 ######################################################################## 516 517 -class RegexList(UnorderedList):
518 519 """ 520 Class representing a list of valid regular expression strings. 521 522 This is an unordered list. 523 524 We override the C{append}, C{insert} and C{extend} methods to ensure that 525 any item added to the list is a valid regular expression. 526 """ 527
528 - def append(self, item):
529 """ 530 Overrides the standard C{append} method. 531 @raise ValueError: If item is not an absolute path. 532 """ 533 try: 534 re.compile(item) 535 except re.error: 536 raise ValueError("Not a valid regular expression: [%s]" % item) 537 list.append(self, item)
538
539 - def insert(self, index, item):
540 """ 541 Overrides the standard C{insert} method. 542 @raise ValueError: If item is not an absolute path. 543 """ 544 try: 545 re.compile(item) 546 except re.error: 547 raise ValueError("Not a valid regular expression: [%s]" % item) 548 list.insert(self, index, item)
549
550 - def extend(self, seq):
551 """ 552 Overrides the standard C{insert} method. 553 @raise ValueError: If any item is not an absolute path. 554 """ 555 for item in seq: 556 try: 557 re.compile(item) 558 except re.error: 559 raise ValueError("Not a valid regular expression: [%s]" % item) 560 for item in seq: 561 list.append(self, item)
562
563 564 ######################################################################## 565 # Directed graph implementation 566 ######################################################################## 567 568 -class _Vertex(object):
569 570 """ 571 Represents a vertex (or node) in a directed graph. 572 """ 573
574 - def __init__(self, name):
575 """ 576 Constructor. 577 @param name: Name of this graph vertex. 578 @type name: String value. 579 """ 580 self.name = name 581 self.endpoints = [] 582 self.state = None
583
584 -class DirectedGraph(object):
585 586 """ 587 Represents a directed graph. 588 589 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set 590 B{E} of vertex pairs or edges. In a directed graph, each edge also has an 591 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph} 592 object provides a way to construct a directed graph and execute a depth- 593 first search. 594 595 This data structure was designed based on the graphing chapter in 596 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>}, 597 by Steven S. Skiena. 598 599 This class is intended to be used by Cedar Backup for dependency ordering. 600 Because of this, it's not quite general-purpose. Unlike a "general" graph, 601 every vertex in this graph has at least one edge pointing to it, from a 602 special "start" vertex. This is so no vertices get "lost" either because 603 they have no dependencies or because nothing depends on them. 604 """ 605 606 _UNDISCOVERED = 0 607 _DISCOVERED = 1 608 _EXPLORED = 2 609
610 - def __init__(self, name):
611 """ 612 Directed graph constructor. 613 614 @param name: Name of this graph. 615 @type name: String value. 616 """ 617 if name is None or name == "": 618 raise ValueError("Graph name must be non-empty.") 619 self._name = name 620 self._vertices = {} 621 self._startVertex = _Vertex(None) # start vertex is only vertex with no name
622
623 - def __repr__(self):
624 """ 625 Official string representation for class instance. 626 """ 627 return "DirectedGraph(%s)" % self.name
628
629 - def __str__(self):
630 """ 631 Informal string representation for class instance. 632 """ 633 return self.__repr__()
634
635 - def __cmp__(self, other):
636 """ 637 Definition of equals operator for this class. 638 @param other: Other object to compare to. 639 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 640 """ 641 # pylint: disable=W0212 642 if other is None: 643 return 1 644 if self.name != other.name: 645 if self.name < other.name: 646 return -1 647 else: 648 return 1 649 if self._vertices != other._vertices: 650 if self._vertices < other._vertices: 651 return -1 652 else: 653 return 1 654 return 0
655
656 - def _getName(self):
657 """ 658 Property target used to get the graph name. 659 """ 660 return self._name
661 662 name = property(_getName, None, None, "Name of the graph.") 663
664 - def createVertex(self, name):
665 """ 666 Creates a named vertex. 667 @param name: vertex name 668 @raise ValueError: If the vertex name is C{None} or empty. 669 """ 670 if name is None or name == "": 671 raise ValueError("Vertex name must be non-empty.") 672 vertex = _Vertex(name) 673 self._startVertex.endpoints.append(vertex) # so every vertex is connected at least once 674 self._vertices[name] = vertex
675
676 - def createEdge(self, start, finish):
677 """ 678 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex. 679 @param start: Name of start vertex. 680 @param finish: Name of finish vertex. 681 @raise ValueError: If one of the named vertices is unknown. 682 """ 683 try: 684 startVertex = self._vertices[start] 685 finishVertex = self._vertices[finish] 686 startVertex.endpoints.append(finishVertex) 687 except KeyError, e: 688 raise ValueError("Vertex [%s] could not be found." % e)
689
690 - def topologicalSort(self):
691 """ 692 Implements a topological sort of the graph. 693 694 This method also enforces that the graph is a directed acyclic graph, 695 which is a requirement of a topological sort. 696 697 A directed acyclic graph (or "DAG") is a directed graph with no directed 698 cycles. A topological sort of a DAG is an ordering on the vertices such 699 that all edges go from left to right. Only an acyclic graph can have a 700 topological sort, but any DAG has at least one topological sort. 701 702 Since a topological sort only makes sense for an acyclic graph, this 703 method throws an exception if a cycle is found. 704 705 A depth-first search only makes sense if the graph is acyclic. If the 706 graph contains any cycles, it is not possible to determine a consistent 707 ordering for the vertices. 708 709 @note: If a particular vertex has no edges, then its position in the 710 final list depends on the order in which the vertices were created in the 711 graph. If you're using this method to determine a dependency order, this 712 makes sense: a vertex with no dependencies can go anywhere (and will). 713 714 @return: Ordering on the vertices so that all edges go from left to right. 715 716 @raise ValueError: If a cycle is found in the graph. 717 """ 718 ordering = [] 719 for key in self._vertices: 720 vertex = self._vertices[key] 721 vertex.state = self._UNDISCOVERED 722 for key in self._vertices: 723 vertex = self._vertices[key] 724 if vertex.state == self._UNDISCOVERED: 725 self._topologicalSort(self._startVertex, ordering) 726 return ordering
727
728 - def _topologicalSort(self, vertex, ordering):
729 """ 730 Recursive depth first search function implementing topological sort. 731 @param vertex: Vertex to search 732 @param ordering: List of vertices in proper order 733 """ 734 vertex.state = self._DISCOVERED 735 for endpoint in vertex.endpoints: 736 if endpoint.state == self._UNDISCOVERED: 737 self._topologicalSort(endpoint, ordering) 738 elif endpoint.state != self._EXPLORED: 739 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name)) 740 if vertex.name is not None: 741 ordering.insert(0, vertex.name) 742 vertex.state = self._EXPLORED
743
744 745 ######################################################################## 746 # PathResolverSingleton class definition 747 ######################################################################## 748 749 -class PathResolverSingleton(object):
750 751 """ 752 Singleton used for resolving executable paths. 753 754 Various functions throughout Cedar Backup (including extensions) need a way 755 to resolve the path of executables that they use. For instance, the image 756 functionality needs to find the C{mkisofs} executable, and the Subversion 757 extension needs to find the C{svnlook} executable. Cedar Backup's original 758 behavior was to assume that the simple name (C{"svnlook"} or whatever) was 759 available on the caller's C{$PATH}, and to fail otherwise. However, this 760 turns out to be less than ideal, since for instance the root user might not 761 always have executables like C{svnlook} in its path. 762 763 One solution is to specify a path (either via an absolute path or some sort 764 of path insertion or path appending mechanism) that would apply to the 765 C{executeCommand()} function. This is not difficult to implement, but it 766 seem like kind of a "big hammer" solution. Besides that, it might also 767 represent a security flaw (for instance, I prefer not to mess with root's 768 C{$PATH} on the application level if I don't have to). 769 770 The alternative is to set up some sort of configuration for the path to 771 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or 772 whatever. This PathResolverSingleton aims to provide a good solution to the 773 mapping problem. Callers of all sorts (extensions or not) can get an 774 instance of the singleton. Then, they call the C{lookup} method to try and 775 resolve the executable they are looking for. Through the C{lookup} method, 776 the caller can also specify a default to use if a mapping is not found. 777 This way, with no real effort on the part of the caller, behavior can neatly 778 degrade to something equivalent to the current behavior if there is no 779 special mapping or if the singleton was never initialized in the first 780 place. 781 782 Even better, extensions automagically get access to the same resolver 783 functionality, and they don't even need to understand how the mapping 784 happens. All extension authors need to do is document what executables 785 their code requires, and the standard resolver configuration section will 786 meet their needs. 787 788 The class should be initialized once through the constructor somewhere in 789 the main routine. Then, the main routine should call the L{fill} method to 790 fill in the resolver's internal structures. Everyone else who needs to 791 resolve a path will get an instance of the class using L{getInstance} and 792 will then just call the L{lookup} method. 793 794 @cvar _instance: Holds a reference to the singleton 795 @ivar _mapping: Internal mapping from resource name to path. 796 """ 797 798 _instance = None # Holds a reference to singleton instance 799
800 - class _Helper(object):
801 """Helper class to provide a singleton factory method."""
802 - def __init__(self):
803 pass
804 - def __call__(self, *args, **kw):
805 # pylint: disable=W0212,R0201 806 if PathResolverSingleton._instance is None: 807 obj = PathResolverSingleton() 808 PathResolverSingleton._instance = obj 809 return PathResolverSingleton._instance
810 811 getInstance = _Helper() # Method that callers will use to get an instance 812
813 - def __init__(self):
814 """Singleton constructor, which just creates the singleton instance.""" 815 if PathResolverSingleton._instance is not None: 816 raise RuntimeError("Only one instance of PathResolverSingleton is allowed!") 817 PathResolverSingleton._instance = self 818 self._mapping = { }
819
820 - def lookup(self, name, default=None):
821 """ 822 Looks up name and returns the resolved path associated with the name. 823 @param name: Name of the path resource to resolve. 824 @param default: Default to return if resource cannot be resolved. 825 @return: Resolved path associated with name, or default if name can't be resolved. 826 """ 827 value = default 828 if name in self._mapping.keys(): 829 value = self._mapping[name] 830 logger.debug("Resolved command [%s] to [%s].", name, value) 831 return value
832
833 - def fill(self, mapping):
834 """ 835 Fills in the singleton's internal mapping from name to resource. 836 @param mapping: Mapping from resource name to path. 837 @type mapping: Dictionary mapping name to path, both as strings. 838 """ 839 self._mapping = { } 840 for key in mapping.keys(): 841 self._mapping[key] = mapping[key]
842
843 844 ######################################################################## 845 # Pipe class definition 846 ######################################################################## 847 848 -class Pipe(Popen):
849 """ 850 Specialized pipe class for use by C{executeCommand}. 851 852 The L{executeCommand} function needs a specialized way of interacting 853 with a pipe. First, C{executeCommand} only reads from the pipe, and 854 never writes to it. Second, C{executeCommand} needs a way to discard all 855 output written to C{stderr}, as a means of simulating the shell 856 C{2>/dev/null} construct. 857 """
858 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
859 stderr = STDOUT 860 if ignoreStderr: 861 devnull = nullDevice() 862 stderr = os.open(devnull, os.O_RDWR) 863 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
864
865 866 ######################################################################## 867 # Diagnostics class definition 868 ######################################################################## 869 870 -class Diagnostics(object):
871 872 """ 873 Class holding runtime diagnostic information. 874 875 Diagnostic information is information that is useful to get from users for 876 debugging purposes. I'm consolidating it all here into one object. 877 878 @sort: __init__, __repr__, __str__ 879 """ 880 # pylint: disable=R0201 881
882 - def __init__(self):
883 """ 884 Constructor for the C{Diagnostics} class. 885 """
886
887 - def __repr__(self):
888 """ 889 Official string representation for class instance. 890 """ 891 return "Diagnostics()"
892
893 - def __str__(self):
894 """ 895 Informal string representation for class instance. 896 """ 897 return self.__repr__()
898
899 - def getValues(self):
900 """ 901 Get a map containing all of the diagnostic values. 902 @return: Map from diagnostic name to diagnostic value. 903 """ 904 values = {} 905 values['version'] = self.version 906 values['interpreter'] = self.interpreter 907 values['platform'] = self.platform 908 values['encoding'] = self.encoding 909 values['locale'] = self.locale 910 values['timestamp'] = self.timestamp 911 values['supported'] = self.supported 912 return values
913
914 - def printDiagnostics(self, fd=sys.stdout, prefix=""):
915 """ 916 Pretty-print diagnostic information to a file descriptor. 917 @param fd: File descriptor used to print information. 918 @param prefix: Prefix string (if any) to place onto printed lines 919 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 920 """ 921 lines = self._buildDiagnosticLines(prefix) 922 for line in lines: 923 fd.write("%s\n" % line)
924
925 - def logDiagnostics(self, method, prefix=""):
926 """ 927 Pretty-print diagnostic information using a logger method. 928 @param method: Logger method to use for logging (i.e. logger.info) 929 @param prefix: Prefix string (if any) to place onto printed lines 930 """ 931 lines = self._buildDiagnosticLines(prefix) 932 for line in lines: 933 method("%s" % line)
934
935 - def _buildDiagnosticLines(self, prefix=""):
936 """ 937 Build a set of pretty-printed diagnostic lines. 938 @param prefix: Prefix string (if any) to place onto printed lines 939 @return: List of strings, not terminated by newlines. 940 """ 941 values = self.getValues() 942 keys = values.keys() 943 keys.sort() 944 tmax = Diagnostics._getMaxLength(keys) + 3 # three extra dots in output 945 lines = [] 946 for key in keys: 947 title = key.title() 948 title += (tmax - len(title)) * '.' 949 value = values[key] 950 line = "%s%s: %s" % (prefix, title, value) 951 lines.append(line) 952 return lines
953 954 @staticmethod
955 - def _getMaxLength(values):
956 """ 957 Get the maximum length from among a list of strings. 958 """ 959 tmax = 0 960 for value in values: 961 if len(value) > tmax: 962 tmax = len(value) 963 return tmax
964
965 - def _getVersion(self):
966 """ 967 Property target to get the Cedar Backup version. 968 """ 969 return "Cedar Backup %s (%s)" % (VERSION, DATE)
970
971 - def _getInterpreter(self):
972 """ 973 Property target to get the Python interpreter version. 974 """ 975 version = sys.version_info 976 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
977
978 - def _getEncoding(self):
979 """ 980 Property target to get the filesystem encoding. 981 """ 982 return sys.getfilesystemencoding() or sys.getdefaultencoding()
983
984 - def _getPlatform(self):
985 """ 986 Property target to get the operating system platform. 987 """ 988 try: 989 if sys.platform.startswith("win"): 990 windowsPlatforms = [ "Windows 3.1", "Windows 95/98/ME", "Windows NT/2000/XP", "Windows CE", ] 991 wininfo = sys.getwindowsversion() # pylint: disable=E1101 992 winversion = "%d.%d.%d" % (wininfo[0], wininfo[1], wininfo[2]) 993 winplatform = windowsPlatforms[wininfo[3]] 994 wintext = wininfo[4] # i.e. "Service Pack 2" 995 return "%s (%s %s %s)" % (sys.platform, winplatform, winversion, wintext) 996 else: 997 uname = os.uname() 998 sysname = uname[0] # i.e. Linux 999 release = uname[2] # i.e. 2.16.18-2 1000 machine = uname[4] # i.e. i686 1001 return "%s (%s %s %s)" % (sys.platform, sysname, release, machine) 1002 except: 1003 return sys.platform
1004
1005 - def _getLocale(self):
1006 """ 1007 Property target to get the default locale that is in effect. 1008 """ 1009 try: 1010 import locale 1011 return locale.getdefaultlocale()[0] 1012 except: 1013 return "(unknown)"
1014
1015 - def _getTimestamp(self):
1016 """ 1017 Property target to get a current date/time stamp. 1018 """ 1019 try: 1020 import datetime 1021 return datetime.datetime.utcnow().ctime() + " UTC" 1022 except: 1023 return "(unknown)"
1024
1025 - def _getSupported(self):
1026 """ 1027 Property target to get the supported value. 1028 """ 1029 return "Unsupported as of 11 Nov 2017"
1030 1031 version = property(_getVersion, None, None, "Cedar Backup version.") 1032 interpreter = property(_getInterpreter, None, None, "Python interpreter version.") 1033 platform = property(_getPlatform, None, None, "Platform identifying information.") 1034 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.") 1035 locale = property(_getLocale, None, None, "Locale that is in effect.") 1036 timestamp = property(_getTimestamp, None, None, "Current timestamp.") 1037 supported = property(_getSupported, None, None, "Whether the version is supported.")
1038
1039 1040 ######################################################################## 1041 # General utility functions 1042 ######################################################################## 1043 1044 ###################### 1045 # sortDict() function 1046 ###################### 1047 1048 -def sortDict(d):
1049 """ 1050 Returns the keys of the dictionary sorted by value. 1051 1052 There are cuter ways to do this in Python 2.4, but we were originally 1053 attempting to stay compatible with Python 2.3. 1054 1055 @param d: Dictionary to operate on 1056 @return: List of dictionary keys sorted in order by dictionary value. 1057 """ 1058 items = d.items() 1059 items.sort(lambda x, y: cmp(x[1], y[1])) 1060 return [key for key, value in items]
1061
1062 1063 ######################## 1064 # removeKeys() function 1065 ######################## 1066 1067 -def removeKeys(d, keys):
1068 """ 1069 Removes all of the keys from the dictionary. 1070 The dictionary is altered in-place. 1071 Each key must exist in the dictionary. 1072 @param d: Dictionary to operate on 1073 @param keys: List of keys to remove 1074 @raise KeyError: If one of the keys does not exist 1075 """ 1076 for key in keys: 1077 del d[key]
1078
1079 1080 ######################### 1081 # convertSize() function 1082 ######################### 1083 1084 -def convertSize(size, fromUnit, toUnit):
1085 """ 1086 Converts a size in one unit to a size in another unit. 1087 1088 This is just a convenience function so that the functionality can be 1089 implemented in just one place. Internally, we convert values to bytes and 1090 then to the final unit. 1091 1092 The available units are: 1093 1094 - C{UNIT_BYTES} - Bytes 1095 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B 1096 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB 1097 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB 1098 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B 1099 1100 @param size: Size to convert 1101 @type size: Integer or float value in units of C{fromUnit} 1102 1103 @param fromUnit: Unit to convert from 1104 @type fromUnit: One of the units listed above 1105 1106 @param toUnit: Unit to convert to 1107 @type toUnit: One of the units listed above 1108 1109 @return: Number converted to new unit, as a float. 1110 @raise ValueError: If one of the units is invalid. 1111 """ 1112 if size is None: 1113 raise ValueError("Cannot convert size of None.") 1114 if fromUnit == UNIT_BYTES: 1115 byteSize = float(size) 1116 elif fromUnit == UNIT_KBYTES: 1117 byteSize = float(size) * BYTES_PER_KBYTE 1118 elif fromUnit == UNIT_MBYTES: 1119 byteSize = float(size) * BYTES_PER_MBYTE 1120 elif fromUnit == UNIT_GBYTES: 1121 byteSize = float(size) * BYTES_PER_GBYTE 1122 elif fromUnit == UNIT_SECTORS: 1123 byteSize = float(size) * BYTES_PER_SECTOR 1124 else: 1125 raise ValueError("Unknown 'from' unit %s." % fromUnit) 1126 if toUnit == UNIT_BYTES: 1127 return byteSize 1128 elif toUnit == UNIT_KBYTES: 1129 return byteSize / BYTES_PER_KBYTE 1130 elif toUnit == UNIT_MBYTES: 1131 return byteSize / BYTES_PER_MBYTE 1132 elif toUnit == UNIT_GBYTES: 1133 return byteSize / BYTES_PER_GBYTE 1134 elif toUnit == UNIT_SECTORS: 1135 return byteSize / BYTES_PER_SECTOR 1136 else: 1137 raise ValueError("Unknown 'to' unit %s." % toUnit)
1138
1139 1140 ########################## 1141 # displayBytes() function 1142 ########################## 1143 1144 -def displayBytes(bytes, digits=2): # pylint: disable=W0622
1145 """ 1146 Format a byte quantity so it can be sensibly displayed. 1147 1148 It's rather difficult to look at a number like "72372224 bytes" and get any 1149 meaningful information out of it. It would be more useful to see something 1150 like "69.02 MB". That's what this function does. Any time you want to display 1151 a byte value, i.e.:: 1152 1153 print "Size: %s bytes" % bytes 1154 1155 Call this function instead:: 1156 1157 print "Size: %s" % displayBytes(bytes) 1158 1159 What comes out will be sensibly formatted. The indicated number of digits 1160 will be listed after the decimal point, rounded based on whatever rules are 1161 used by Python's standard C{%f} string format specifier. (Values less than 1 1162 kB will be listed in bytes and will not have a decimal point, since the 1163 concept of a fractional byte is nonsensical.) 1164 1165 @param bytes: Byte quantity. 1166 @type bytes: Integer number of bytes. 1167 1168 @param digits: Number of digits to display after the decimal point. 1169 @type digits: Integer value, typically 2-5. 1170 1171 @return: String, formatted for sensible display. 1172 """ 1173 if bytes is None: 1174 raise ValueError("Cannot display byte value of None.") 1175 bytes = float(bytes) 1176 if math.fabs(bytes) < BYTES_PER_KBYTE: 1177 fmt = "%.0f bytes" 1178 value = bytes 1179 elif math.fabs(bytes) < BYTES_PER_MBYTE: 1180 fmt = "%." + "%d" % digits + "f kB" 1181 value = bytes / BYTES_PER_KBYTE 1182 elif math.fabs(bytes) < BYTES_PER_GBYTE: 1183 fmt = "%." + "%d" % digits + "f MB" 1184 value = bytes / BYTES_PER_MBYTE 1185 else: 1186 fmt = "%." + "%d" % digits + "f GB" 1187 value = bytes / BYTES_PER_GBYTE 1188 return fmt % value 1189
1190 1191 ################################## 1192 # getFunctionReference() function 1193 ################################## 1194 1195 -def getFunctionReference(module, function):
1196 """ 1197 Gets a reference to a named function. 1198 1199 This does some hokey-pokey to get back a reference to a dynamically named 1200 function. For instance, say you wanted to get a reference to the 1201 C{os.path.isdir} function. You could use:: 1202 1203 myfunc = getFunctionReference("os.path", "isdir") 1204 1205 Although we won't bomb out directly, behavior is pretty much undefined if 1206 you pass in C{None} or C{""} for either C{module} or C{function}. 1207 1208 The only validation we enforce is that whatever we get back must be 1209 callable. 1210 1211 I derived this code based on the internals of the Python unittest 1212 implementation. I don't claim to completely understand how it works. 1213 1214 @param module: Name of module associated with function. 1215 @type module: Something like "os.path" or "CedarBackup2.util" 1216 1217 @param function: Name of function 1218 @type function: Something like "isdir" or "getUidGid" 1219 1220 @return: Reference to function associated with name. 1221 1222 @raise ImportError: If the function cannot be found. 1223 @raise ValueError: If the resulting reference is not callable. 1224 1225 @copyright: Some of this code, prior to customization, was originally part 1226 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python 1227 Software Foundation; All Rights Reserved. 1228 """ 1229 parts = [] 1230 if module is not None and module != "": 1231 parts = module.split(".") 1232 if function is not None and function != "": 1233 parts.append(function) 1234 copy = parts[:] 1235 while copy: 1236 try: 1237 module = __import__(string.join(copy, ".")) 1238 break 1239 except ImportError: 1240 del copy[-1] 1241 if not copy: raise 1242 parts = parts[1:] 1243 obj = module 1244 for part in parts: 1245 obj = getattr(obj, part) 1246 if not callable(obj): 1247 raise ValueError("Reference to %s.%s is not callable." % (module, function)) 1248 return obj
1249
1250 1251 ####################### 1252 # getUidGid() function 1253 ####################### 1254 1255 -def getUidGid(user, group):
1256 """ 1257 Get the uid/gid associated with a user/group pair 1258 1259 This is a no-op if user/group functionality is not available on the platform. 1260 1261 @param user: User name 1262 @type user: User name as a string 1263 1264 @param group: Group name 1265 @type group: Group name as a string 1266 1267 @return: Tuple C{(uid, gid)} matching passed-in user and group. 1268 @raise ValueError: If the ownership user/group values are invalid 1269 """ 1270 if _UID_GID_AVAILABLE: 1271 try: 1272 uid = pwd.getpwnam(user)[2] 1273 gid = grp.getgrnam(group)[2] 1274 return (uid, gid) 1275 except Exception, e: 1276 logger.debug("Error looking up uid and gid for [%s:%s]: %s", user, group, e) 1277 raise ValueError("Unable to lookup up uid and gid for passed in user/group.") 1278 else: 1279 return (0, 0)
1280
1281 1282 ############################# 1283 # changeOwnership() function 1284 ############################# 1285 1286 -def changeOwnership(path, user, group):
1287 """ 1288 Changes ownership of path to match the user and group. 1289 1290 This is a no-op if user/group functionality is not available on the 1291 platform, or if the either passed-in user or group is C{None}. Further, we 1292 won't even try to do it unless running as root, since it's unlikely to work. 1293 1294 @param path: Path whose ownership to change. 1295 @param user: User which owns file. 1296 @param group: Group which owns file. 1297 """ 1298 if _UID_GID_AVAILABLE: 1299 if user is None or group is None: 1300 logger.debug("User or group is None, so not attempting to change owner on [%s].", path) 1301 elif not isRunningAsRoot(): 1302 logger.debug("Not root, so not attempting to change owner on [%s].", path) 1303 else: 1304 try: 1305 (uid, gid) = getUidGid(user, group) 1306 os.chown(path, uid, gid) 1307 except Exception, e: 1308 logger.error("Error changing ownership of [%s]: %s", path, e)
1309
1310 1311 ############################# 1312 # isRunningAsRoot() function 1313 ############################# 1314 1315 -def isRunningAsRoot():
1316 """ 1317 Indicates whether the program is running as the root user. 1318 """ 1319 return os.getuid() == 0
1320
1321 1322 ############################## 1323 # splitCommandLine() function 1324 ############################## 1325 1326 -def splitCommandLine(commandLine):
1327 """ 1328 Splits a command line string into a list of arguments. 1329 1330 Unfortunately, there is no "standard" way to parse a command line string, 1331 and it's actually not an easy problem to solve portably (essentially, we 1332 have to emulate the shell argument-processing logic). This code only 1333 respects double quotes (C{"}) for grouping arguments, not single quotes 1334 (C{'}). Make sure you take this into account when building your command 1335 line. 1336 1337 Incidentally, I found this particular parsing method while digging around in 1338 Google Groups, and I tweaked it for my own use. 1339 1340 @param commandLine: Command line string 1341 @type commandLine: String, i.e. "cback --verbose stage store" 1342 1343 @return: List of arguments, suitable for passing to C{popen2}. 1344 1345 @raise ValueError: If the command line is None. 1346 """ 1347 if commandLine is None: 1348 raise ValueError("Cannot split command line of None.") 1349 fields = re.findall('[^ "]+|"[^"]+"', commandLine) 1350 fields = [field.replace('"', '') for field in fields] 1351 return fields
1352
1353 1354 ############################ 1355 # resolveCommand() function 1356 ############################ 1357 1358 -def resolveCommand(command):
1359 """ 1360 Resolves the real path to a command through the path resolver mechanism. 1361 1362 Both extensions and standard Cedar Backup functionality need a way to 1363 resolve the "real" location of various executables. Normally, they assume 1364 that these executables are on the system path, but some callers need to 1365 specify an alternate location. 1366 1367 Ideally, we want to handle this configuration in a central location. The 1368 Cedar Backup path resolver mechanism (a singleton called 1369 L{PathResolverSingleton}) provides the central location to store the 1370 mappings. This function wraps access to the singleton, and is what all 1371 functions (extensions or standard functionality) should call if they need to 1372 find a command. 1373 1374 The passed-in command must actually be a list, in the standard form used by 1375 all existing Cedar Backup code (something like C{["svnlook", ]}). The 1376 lookup will actually be done on the first element in the list, and the 1377 returned command will always be in list form as well. 1378 1379 If the passed-in command can't be resolved or no mapping exists, then the 1380 command itself will be returned unchanged. This way, we neatly fall back on 1381 default behavior if we have no sensible alternative. 1382 1383 @param command: Command to resolve. 1384 @type command: List form of command, i.e. C{["svnlook", ]}. 1385 1386 @return: Path to command or just command itself if no mapping exists. 1387 """ 1388 singleton = PathResolverSingleton.getInstance() 1389 name = command[0] 1390 result = command[:] 1391 result[0] = singleton.lookup(name, name) 1392 return result
1393
1394 1395 ############################ 1396 # executeCommand() function 1397 ############################ 1398 1399 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1400 """ 1401 Executes a shell command, hopefully in a safe way. 1402 1403 This function exists to replace direct calls to C{os.popen} in the Cedar 1404 Backup code. It's not safe to call a function such as C{os.popen()} with 1405 untrusted arguments, since that can cause problems if the string contains 1406 non-safe variables or other constructs (imagine that the argument is 1407 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/; 1408 echo"} in the current environment). 1409 1410 Instead, it's safer to pass a list of arguments in the style supported bt 1411 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe} 1412 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}. 1413 1414 Under the normal case, this function will return a tuple of C{(status, 1415 None)} where the status is the wait-encoded return status of the call per 1416 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as 1417 C{True}, the function will return a tuple of C{(status, output)} where 1418 C{output} is a list of strings, one entry per line in the output from the 1419 command. Output is always logged to the C{outputLogger.info()} target, 1420 regardless of whether it's returned. 1421 1422 By default, C{stdout} and C{stderr} will be intermingled in the output. 1423 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be 1424 included in the output. 1425 1426 The C{doNotLog} parameter exists so that callers can force the function to 1427 not log command output to the debug log. Normally, you would want to log. 1428 However, if you're using this function to write huge output files (i.e. 1429 database backups written to C{stdout}) then you might want to avoid putting 1430 all that information into the debug log. 1431 1432 The C{outputFile} parameter exists to make it easier for a caller to push 1433 output into a file, i.e. as a substitute for redirection to a file. If this 1434 value is passed in, each time a line of output is generated, it will be 1435 written to the file using C{outputFile.write()}. At the end, the file 1436 descriptor will be flushed using C{outputFile.flush()}. The caller 1437 maintains responsibility for closing the file object appropriately. 1438 1439 @note: I know that it's a bit confusing that the command and the arguments 1440 are both lists. I could have just required the caller to pass in one big 1441 list. However, I think it makes some sense to keep the command (the 1442 constant part of what we're executing, i.e. C{"scp -B"}) separate from its 1443 arguments, even if they both end up looking kind of similar. 1444 1445 @note: You cannot redirect output via shell constructs (i.e. C{>file}, 1446 C{2>/dev/null}, etc.) using this function. The redirection string would be 1447 passed to the command just like any other argument. However, you can 1448 implement the equivalent to redirection using C{ignoreStderr} and 1449 C{outputFile}, as discussed above. 1450 1451 @note: The operating system environment is partially sanitized before 1452 the command is invoked. See L{sanitizeEnvironment} for details. 1453 1454 @param command: Shell command to execute 1455 @type command: List of individual arguments that make up the command 1456 1457 @param args: List of arguments to the command 1458 @type args: List of additional arguments to the command 1459 1460 @param returnOutput: Indicates whether to return the output of the command 1461 @type returnOutput: Boolean C{True} or C{False} 1462 1463 @param ignoreStderr: Whether stderr should be discarded 1464 @type ignoreStderr: Boolean True or False 1465 1466 @param doNotLog: Indicates that output should not be logged. 1467 @type doNotLog: Boolean C{True} or C{False} 1468 1469 @param outputFile: File object that all output should be written to. 1470 @type outputFile: File object as returned from C{open()} or C{file()}. 1471 1472 @return: Tuple of C{(result, output)} as described above. 1473 """ 1474 logger.debug("Executing command %s with args %s.", command, args) 1475 outputLogger.info("Executing command %s with args %s.", command, args) 1476 if doNotLog: 1477 logger.debug("Note: output will not be logged, per the doNotLog flag.") 1478 outputLogger.info("Note: output will not be logged, per the doNotLog flag.") 1479 output = [] 1480 fields = command[:] # make sure to copy it so we don't destroy it 1481 fields.extend(args) 1482 try: 1483 sanitizeEnvironment() # make sure we have a consistent environment 1484 try: 1485 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1486 except OSError: 1487 # On some platforms (i.e. Cygwin) this intermittently fails the first time we do it. 1488 # So, we attempt it a second time and if that works, we just go on as usual. 1489 # The problem appears to be that we sometimes get a bad stderr file descriptor. 1490 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1491 while True: 1492 line = pipe.stdout.readline() 1493 if not line: break 1494 if returnOutput: output.append(line) 1495 if outputFile is not None: outputFile.write(line) 1496 if not doNotLog: outputLogger.info(line[:-1]) # this way the log will (hopefully) get updated in realtime 1497 if outputFile is not None: 1498 try: # note, not every file-like object can be flushed 1499 outputFile.flush() 1500 except: pass 1501 if returnOutput: 1502 return (pipe.wait(), output) 1503 else: 1504 return (pipe.wait(), None) 1505 except OSError, e: 1506 try: 1507 if returnOutput: 1508 if output != []: 1509 return (pipe.wait(), output) 1510 else: 1511 return (pipe.wait(), [ e, ]) 1512 else: 1513 return (pipe.wait(), None) 1514 except UnboundLocalError: # pipe not set 1515 if returnOutput: 1516 return (256, []) 1517 else: 1518 return (256, None)
1519
1520 1521 ############################## 1522 # calculateFileAge() function 1523 ############################## 1524 1525 -def calculateFileAge(path):
1526 """ 1527 Calculates the age (in days) of a file. 1528 1529 The "age" of a file is the amount of time since the file was last used, per 1530 the most recent of the file's C{st_atime} and C{st_mtime} values. 1531 1532 Technically, we only intend this function to work with files, but it will 1533 probably work with anything on the filesystem. 1534 1535 @param path: Path to a file on disk. 1536 1537 @return: Age of the file in days (possibly fractional). 1538 @raise OSError: If the file doesn't exist. 1539 """ 1540 currentTime = int(time.time()) 1541 fileStats = os.stat(path) 1542 lastUse = max(fileStats.st_atime, fileStats.st_mtime) # "most recent" is "largest" 1543 ageInSeconds = currentTime - lastUse 1544 ageInDays = ageInSeconds / SECONDS_PER_DAY 1545 return ageInDays
1546
1547 1548 ################### 1549 # mount() function 1550 ################### 1551 1552 -def mount(devicePath, mountPoint, fsType):
1553 """ 1554 Mounts the indicated device at the indicated mount point. 1555 1556 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount 1557 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any 1558 filesystem type that is supported by C{mount} on your platform. If the type 1559 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may 1560 not work on all systems. 1561 1562 @note: This only works on platforms that have a concept of "mounting" a 1563 filesystem through a command-line C{"mount"} command, like UNIXes. It 1564 won't work on Windows. 1565 1566 @param devicePath: Path of device to be mounted. 1567 @param mountPoint: Path that device should be mounted at. 1568 @param fsType: Type of the filesystem assumed to be available via the device. 1569 1570 @raise IOError: If the device cannot be mounted. 1571 """ 1572 if fsType is None: 1573 args = [ devicePath, mountPoint ] 1574 else: 1575 args = [ "-t", fsType, devicePath, mountPoint ] 1576 command = resolveCommand(MOUNT_COMMAND) 1577 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0] 1578 if result != 0: 1579 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1580
1581 1582 ##################### 1583 # unmount() function 1584 ##################### 1585 1586 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1587 """ 1588 Unmounts whatever device is mounted at the indicated mount point. 1589 1590 Sometimes, it might not be possible to unmount the mount point immediately, 1591 if there are still files open there. Use the C{attempts} and C{waitSeconds} 1592 arguments to indicate how many unmount attempts to make and how many seconds 1593 to wait between attempts. If you pass in zero attempts, no attempts will be 1594 made (duh). 1595 1596 If the indicated mount point is not really a mount point per 1597 C{os.path.ismount()}, then it will be ignored. This seems to be a safer 1598 check then looking through C{/etc/mtab}, since C{ismount()} is already in 1599 the Python standard library and is documented as working on all POSIX 1600 systems. 1601 1602 If C{removeAfter} is C{True}, then the mount point will be removed using 1603 C{os.rmdir()} after the unmount action succeeds. If for some reason the 1604 mount point is not a directory, then it will not be removed. 1605 1606 @note: This only works on platforms that have a concept of "mounting" a 1607 filesystem through a command-line C{"mount"} command, like UNIXes. It 1608 won't work on Windows. 1609 1610 @param mountPoint: Mount point to be unmounted. 1611 @param removeAfter: Remove the mount point after unmounting it. 1612 @param attempts: Number of times to attempt the unmount. 1613 @param waitSeconds: Number of seconds to wait between repeated attempts. 1614 1615 @raise IOError: If the mount point is still mounted after attempts are exhausted. 1616 """ 1617 if os.path.ismount(mountPoint): 1618 for attempt in range(0, attempts): 1619 logger.debug("Making attempt %d to unmount [%s].", attempt, mountPoint) 1620 command = resolveCommand(UMOUNT_COMMAND) 1621 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0] 1622 if result != 0: 1623 logger.error("Error [%d] unmounting [%s] on attempt %d.", result, mountPoint, attempt) 1624 elif os.path.ismount(mountPoint): 1625 logger.error("After attempt %d, [%s] is still mounted.", attempt, mountPoint) 1626 else: 1627 logger.debug("Successfully unmounted [%s] on attempt %d.", mountPoint, attempt) 1628 break # this will cause us to skip the loop else: clause 1629 if attempt+1 < attempts: # i.e. this isn't the last attempt 1630 if waitSeconds > 0: 1631 logger.info("Sleeping %d second(s) before next unmount attempt.", waitSeconds) 1632 time.sleep(waitSeconds) 1633 else: 1634 if os.path.ismount(mountPoint): 1635 raise IOError("Unable to unmount [%s] after %d attempts." % (mountPoint, attempts)) 1636 logger.info("Mount point [%s] seems to have finally gone away.", mountPoint) 1637 if os.path.isdir(mountPoint) and removeAfter: 1638 logger.debug("Removing mount point [%s].", mountPoint) 1639 os.rmdir(mountPoint)
1640
1641 1642 ########################### 1643 # deviceMounted() function 1644 ########################### 1645 1646 -def deviceMounted(devicePath):
1647 """ 1648 Indicates whether a specific filesystem device is currently mounted. 1649 1650 We determine whether the device is mounted by looking through the system's 1651 C{mtab} file. This file shows every currently-mounted filesystem, ordered 1652 by device. We only do the check if the C{mtab} file exists and is readable. 1653 Otherwise, we assume that the device is not mounted. 1654 1655 @note: This only works on platforms that have a concept of an mtab file 1656 to show mounted volumes, like UNIXes. It won't work on Windows. 1657 1658 @param devicePath: Path of device to be checked 1659 1660 @return: True if device is mounted, false otherwise. 1661 """ 1662 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK): 1663 realPath = os.path.realpath(devicePath) 1664 lines = open(MTAB_FILE).readlines() 1665 for line in lines: 1666 (mountDevice, mountPoint, remainder) = line.split(None, 2) 1667 if mountDevice in [ devicePath, realPath, ]: 1668 logger.debug("Device [%s] is mounted at [%s].", devicePath, mountPoint) 1669 return True 1670 return False
1671
1672 1673 ######################## 1674 # encodePath() function 1675 ######################## 1676 1677 -def encodePath(path):
1678 1679 r""" 1680 Safely encodes a filesystem path. 1681 1682 Many Python filesystem functions, such as C{os.listdir}, behave differently 1683 if they are passed unicode arguments versus simple string arguments. For 1684 instance, C{os.listdir} generally returns unicode path names if it is passed 1685 a unicode argument, and string pathnames if it is passed a string argument. 1686 1687 However, this behavior often isn't as consistent as we might like. As an example, 1688 C{os.listdir} "gives up" if it finds a filename that it can't properly encode 1689 given the current locale settings. This means that the returned list is 1690 a mixed set of unicode and simple string paths. This has consequences later, 1691 because other filesystem functions like C{os.path.join} will blow up if they 1692 are given one string path and one unicode path. 1693 1694 On comp.lang.python, Martin v. Löwis explained the C{os.listdir} behavior 1695 like this:: 1696 1697 The operating system (POSIX) does not have the inherent notion that file 1698 names are character strings. Instead, in POSIX, file names are primarily 1699 byte strings. There are some bytes which are interpreted as characters 1700 (e.g. '\x2e', which is '.', or '\x2f', which is '/'), but apart from 1701 that, most OS layers think these are just bytes. 1702 1703 Now, most *people* think that file names are character strings. To 1704 interpret a file name as a character string, you need to know what the 1705 encoding is to interpret the file names (which are byte strings) as 1706 character strings. 1707 1708 There is, unfortunately, no operating system API to carry the notion of a 1709 file system encoding. By convention, the locale settings should be used 1710 to establish this encoding, in particular the LC_CTYPE facet of the 1711 locale. This is defined in the environment variables LC_CTYPE, LC_ALL, 1712 and LANG (searched in this order). 1713 1714 If LANG is not set, the "C" locale is assumed, which uses ASCII as its 1715 file system encoding. In this locale, '\xe2\x99\xaa\xe2\x99\xac' is not a 1716 valid file name (at least it cannot be interpreted as characters, and 1717 hence not be converted to Unicode). 1718 1719 Now, your Python script has requested that all file names *should* be 1720 returned as character (ie. Unicode) strings, but Python cannot comply, 1721 since there is no way to find out what this byte string means, in terms 1722 of characters. 1723 1724 So we have three options: 1725 1726 1. Skip this string, only return the ones that can be converted to Unicode. 1727 Give the user the impression the file does not exist. 1728 2. Return the string as a byte string 1729 3. Refuse to listdir altogether, raising an exception (i.e. return nothing) 1730 1731 Python has chosen alternative 2, allowing the application to implement 1 1732 or 3 on top of that if it wants to (or come up with other strategies, 1733 such as user feedback). 1734 1735 As a solution, he suggests that rather than passing unicode paths into the 1736 filesystem functions, that I should sensibly encode the path first. That is 1737 what this function accomplishes. Any function which takes a filesystem path 1738 as an argument should encode it first, before using it for any other purpose. 1739 1740 I confess I still don't completely understand how this works. On a system 1741 with filesystem encoding "ISO-8859-1", a path C{u"\xe2\x99\xaa\xe2\x99\xac"} 1742 is converted into the string C{"\xe2\x99\xaa\xe2\x99\xac"}. However, on a 1743 system with a "utf-8" encoding, the result is a completely different string: 1744 C{"\xc3\xa2\xc2\x99\xc2\xaa\xc3\xa2\xc2\x99\xc2\xac"}. A quick test where I 1745 write to the first filename and open the second proves that the two strings 1746 represent the same file on disk, which is all I really care about. 1747 1748 @note: As a special case, if C{path} is C{None}, then this function will 1749 return C{None}. 1750 1751 @note: To provide several examples of encoding values, my Debian sarge box 1752 with an ext3 filesystem has Python filesystem encoding C{ISO-8859-1}. User 1753 Anarcat's Debian box with a xfs filesystem has filesystem encoding 1754 C{ANSI_X3.4-1968}. Both my iBook G4 running Mac OS X 10.4 and user Dag 1755 Rende's SuSE 9.3 box both have filesystem encoding C{UTF-8}. 1756 1757 @note: Just because a filesystem has C{UTF-8} encoding doesn't mean that it 1758 will be able to handle all extended-character filenames. For instance, 1759 certain extended-character (but not UTF-8) filenames -- like the ones in the 1760 regression test tar file C{test/data/tree13.tar.gz} -- are not valid under 1761 Mac OS X, and it's not even possible to extract them from the tarfile on 1762 that platform. 1763 1764 @param path: Path to encode 1765 1766 @return: Path, as a string, encoded appropriately 1767 @raise ValueError: If the path cannot be encoded properly. 1768 """ 1769 if path is None: 1770 return path 1771 try: 1772 if isinstance(path, unicode): 1773 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 1774 path = path.encode(encoding) 1775 return path 1776 except UnicodeError: 1777 raise ValueError("Path could not be safely encoded as %s." % encoding)
1778
1779 1780 ######################## 1781 # nullDevice() function 1782 ######################## 1783 1784 -def nullDevice():
1785 """ 1786 Attempts to portably return the null device on this system. 1787 1788 The null device is something like C{/dev/null} on a UNIX system. The name 1789 varies on other platforms. 1790 """ 1791 return os.devnull
1792
1793 1794 ############################## 1795 # deriveDayOfWeek() function 1796 ############################## 1797 1798 -def deriveDayOfWeek(dayName):
1799 """ 1800 Converts English day name to numeric day of week as from C{time.localtime}. 1801 1802 For instance, the day C{monday} would be converted to the number C{0}. 1803 1804 @param dayName: Day of week to convert 1805 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1806 1807 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible. 1808 """ 1809 if dayName.lower() == "monday": 1810 return 0 1811 elif dayName.lower() == "tuesday": 1812 return 1 1813 elif dayName.lower() == "wednesday": 1814 return 2 1815 elif dayName.lower() == "thursday": 1816 return 3 1817 elif dayName.lower() == "friday": 1818 return 4 1819 elif dayName.lower() == "saturday": 1820 return 5 1821 elif dayName.lower() == "sunday": 1822 return 6 1823 else: 1824 return -1 # What else can we do?? Thrown an exception, I guess.
1825
1826 1827 ########################### 1828 # isStartOfWeek() function 1829 ########################### 1830 1831 -def isStartOfWeek(startingDay):
1832 """ 1833 Indicates whether "today" is the backup starting day per configuration. 1834 1835 If the current day's English name matches the indicated starting day, then 1836 today is a starting day. 1837 1838 @param startingDay: Configured starting day. 1839 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1840 1841 @return: Boolean indicating whether today is the starting day. 1842 """ 1843 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay) 1844 if value: 1845 logger.debug("Today is the start of the week.") 1846 else: 1847 logger.debug("Today is NOT the start of the week.") 1848 return value
1849
1850 1851 ################################# 1852 # buildNormalizedPath() function 1853 ################################# 1854 1855 -def buildNormalizedPath(path):
1856 """ 1857 Returns a "normalized" path based on a path name. 1858 1859 A normalized path is a representation of a path that is also a valid file 1860 name. To make a valid file name out of a complete path, we have to convert 1861 or remove some characters that are significant to the filesystem -- in 1862 particular, the path separator and any leading C{'.'} character (which would 1863 cause the file to be hidden in a file listing). 1864 1865 Note that this is a one-way transformation -- you can't safely derive the 1866 original path from the normalized path. 1867 1868 To normalize a path, we begin by looking at the first character. If the 1869 first character is C{'/'} or C{'\\'}, it gets removed. If the first 1870 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the 1871 rest of the path and convert all remaining C{'/'} or C{'\\'} characters 1872 C{'-'}, and all remaining whitespace characters to C{'_'}. 1873 1874 As a special case, a path consisting only of a single C{'/'} or C{'\\'} 1875 character will be converted to C{'-'}. 1876 1877 @param path: Path to normalize 1878 1879 @return: Normalized path as described above. 1880 1881 @raise ValueError: If the path is None 1882 """ 1883 if path is None: 1884 raise ValueError("Cannot normalize path None.") 1885 elif len(path) == 0: 1886 return path 1887 elif path == "/" or path == "\\": 1888 return "-" 1889 else: 1890 normalized = path 1891 normalized = re.sub(r"^\/", "", normalized) # remove leading '/' 1892 normalized = re.sub(r"^\\", "", normalized) # remove leading '\' 1893 normalized = re.sub(r"^\.", "_", normalized) # convert leading '.' to '_' so file won't be hidden 1894 normalized = re.sub(r"\/", "-", normalized) # convert all '/' characters to '-' 1895 normalized = re.sub(r"\\", "-", normalized) # convert all '\' characters to '-' 1896 normalized = re.sub(r"\s", "_", normalized) # convert all whitespace to '_' 1897 return normalized
1898
1899 1900 ################################# 1901 # sanitizeEnvironment() function 1902 ################################# 1903 1904 -def sanitizeEnvironment():
1905 """ 1906 Sanitizes the operating system environment. 1907 1908 The operating system environment is contained in C{os.environ}. This method 1909 sanitizes the contents of that dictionary. 1910 1911 Currently, all it does is reset the locale (removing C{$LC_*}) and set the 1912 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count 1913 on consistent localization regardless of what the end-user has configured. 1914 This is important for code that needs to parse program output. 1915 1916 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already 1917 set to the proper value, it is not re-set, so we can avoid the memory leaks 1918 that are documented to occur on BSD-based systems. 1919 1920 @return: Copy of the sanitized environment. 1921 """ 1922 for var in LOCALE_VARS: 1923 if os.environ.has_key(var): 1924 del os.environ[var] 1925 if os.environ.has_key(LANG_VAR): 1926 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE: # no need to reset if it exists (avoid leaks on BSD systems) 1927 os.environ[LANG_VAR] = DEFAULT_LANGUAGE 1928 return os.environ.copy()
1929 1948
1949 1950 ######################### 1951 # checkUnique() function 1952 ######################### 1953 1954 -def checkUnique(prefix, values):
1955 """ 1956 Checks that all values are unique. 1957 1958 The values list is checked for duplicate values. If there are 1959 duplicates, an exception is thrown. All duplicate values are listed in 1960 the exception. 1961 1962 @param prefix: Prefix to use in the thrown exception 1963 @param values: List of values to check 1964 1965 @raise ValueError: If there are duplicates in the list 1966 """ 1967 values.sort() 1968 duplicates = [] 1969 for i in range(1, len(values)): 1970 if values[i-1] == values[i]: 1971 duplicates.append(values[i]) 1972 if duplicates: 1973 raise ValueError("%s %s" % (prefix, duplicates))
1974
1975 1976 ####################################### 1977 # parseCommaSeparatedString() function 1978 ####################################### 1979 1980 -def parseCommaSeparatedString(commaString):
1981 """ 1982 Parses a list of values out of a comma-separated string. 1983 1984 The items in the list are split by comma, and then have whitespace 1985 stripped. As a special case, if C{commaString} is C{None}, then C{None} 1986 will be returned. 1987 1988 @param commaString: List of values in comma-separated string format. 1989 @return: Values from commaString split into a list, or C{None}. 1990 """ 1991 if commaString is None: 1992 return None 1993 else: 1994 pass1 = commaString.split(",") 1995 pass2 = [] 1996 for item in pass1: 1997 item = item.strip() 1998 if len(item) > 0: 1999 pass2.append(item) 2000 return pass2
2001