zeroinstall.injector.iface

56 """A feed that has been downloaded but not yet added to the interface cache. 57 Feeds remain in this state until the user confirms that they trust at least 58 one of the signatures. 59 @ivar url: URL for the feed 60 @type url: str 61 @ivar signed_data: the untrusted data 62 @type signed_data: stream 63 @ivar sigs: signatures extracted from signed_data 64 @type sigs: [L{gpg.Signature}] 65 @ivar new_xml: the payload of the signed_data, or the whole thing if XML 66 @type new_xml: str 67 @since: 0.25""" 68 __slots__ = ['url', 'signed_data', 'sigs', 'new_xml'] 69

70 - def __init__(self, url, signed_data):

71 """Downloaded data is a GPG-signed message. 72 @param url: the URL of the downloaded feed 73 @type url: str 74 @param signed_data: the downloaded data (not yet trusted) 75 @type signed_data: stream 76 @raise SafeException: if the data is not signed, and logs the actual data""" 77 self.url = url 78 self.signed_data = signed_data 79 self.recheck()

80

81 - def download_keys(self, fetcher, feed_hint = None, key_mirror = None):

82 """Download any required GPG keys not already on our keyring. 83 When all downloads are done (successful or otherwise), add any new keys 84 to the keyring, L{recheck}. 85 @param fetcher: fetcher to manage the download (was Handler before version 1.5) 86 @type fetcher: L{fetch.Fetcher} 87 @param key_mirror: URL of directory containing keys, or None to use feed's directory 88 @type key_mirror: str 89 @rtype: [L{zeroinstall.support.tasks.Blocker}]""" 90 downloads = {} 91 blockers = [] 92 for x in self.sigs: 93 key_id = x.need_key() 94 if key_id: 95 try: 96 import urlparse 97 except ImportError: 98 from urllib import parse as urlparse # Python 3 99 key_url = urlparse.urljoin(key_mirror or self.url, '%s.gpg' % key_id) 100 logger.info(_("Fetching key from %s"), key_url) 101 dl = fetcher.download_url(key_url, hint = feed_hint) 102 downloads[dl.downloaded] = (dl, dl.tempfile) 103 blockers.append(dl.downloaded) 104 105 exception = None 106 any_success = False 107 108 from zeroinstall.support import tasks 109 110 while blockers: 111 yield blockers 112 113 old_blockers = blockers 114 blockers = [] 115 116 for b in old_blockers: 117 dl, stream = downloads[b] 118 try: 119 tasks.check(b) 120 if b.happened: 121 stream.seek(0) 122 self._downloaded_key(stream) 123 any_success = True 124 stream.close() 125 else: 126 blockers.append(b) 127 except Exception: 128 _type, exception, tb = sys.exc_info() 129 logger.warning(_("Failed to import key for '%(url)s': %(exception)s"), {'url': self.url, 'exception': str(exception)}) 130 stream.close() 131 132 if exception and not any_success: 133 raise_with_traceback(exception, tb) 134 135 self.recheck()

136

137 - def _downloaded_key(self, stream):

138 """@type stream: file""" 139 import shutil, tempfile 140 from zeroinstall.injector import gpg 141 142 logger.info(_("Importing key for feed '%s'"), self.url) 143 144 # Python2.4: can't call fileno() on stream, so save to tmp file instead 145 tmpfile = tempfile.TemporaryFile(prefix = 'injector-dl-data-') 146 try: 147 shutil.copyfileobj(stream, tmpfile) 148 tmpfile.flush() 149 150 tmpfile.seek(0) 151 gpg.import_key(tmpfile) 152 finally: 153 tmpfile.close()

154

155 - def recheck(self):

156 """Set new_xml and sigs by reading signed_data. 157 You need to call this when previously-missing keys are added to the GPG keyring.""" 158 from . import gpg 159 try: 160 self.signed_data.seek(0) 161 stream, sigs = gpg.check_stream(self.signed_data) 162 assert sigs 163 164 data = stream.read() 165 if stream is not self.signed_data: 166 stream.close() 167 168 self.new_xml = data 169 self.sigs = sigs 170 except: 171 self.signed_data.seek(0) 172 logger.info(_("Failed to check GPG signature. Data received was:\n") + repr(self.signed_data.read())) 173 raise

176 """ 177 The interface cache stores downloaded and verified interfaces in 178 ~/.cache/0install.net/interfaces (by default). 179 180 There are methods to query the cache, add to it, check signatures, etc. 181 182 The cache is updated by L{fetch.Fetcher}. 183 184 Confusingly, this class is really two caches combined: the in-memory 185 cache of L{model.Interface} objects, and an on-disk cache of L{model.ZeroInstallFeed}s. 186 It will probably be split into two in future. 187 188 @ivar distro: the native distribution proxy 189 @type distro: L{distro.Distribution} 190 191 @see: L{iface_cache} - the singleton IfaceCache instance. 192 """ 193 194 __slots__ = ['_interfaces', '_feeds', '_distro', '_config'] 195

196 - def __init__(self, distro = None):

197 """@param distro: distribution used to fetch "distribution:" feeds (since 0.49) 198 @type distro: L{distro.Distribution}, or None to use the host distribution""" 199 self._interfaces = {} 200 self._feeds = {} 201 self._distro = distro

202 203 @property

204 - def stores(self):

205 from zeroinstall.injector import policy 206 return policy.get_deprecated_singleton_config().stores

207 208 @property

209 - def distro(self):

210 if self._distro is None: 211 from zeroinstall.injector.distro import get_host_distribution 212 self._distro = get_host_distribution() 213 return self._distro

214

215 - def update_interface_if_trusted(self, interface, sigs, xml):

216 import warnings 217 warnings.warn("Use update_feed_if_trusted instead", DeprecationWarning, stacklevel = 2) 218 return self.update_feed_if_trusted(interface.uri, sigs, xml)

219

220 - def update_feed_if_trusted(self, feed_url, sigs, xml, dry_run = False):

221 """Update a cached feed (using L{update_feed_from_network}) 222 if we trust the signatures. 223 If we don't trust any of the signatures, do nothing. 224 @param feed_url: the feed being updated 225 @type feed_url: str 226 @param sigs: signatures from L{gpg.check_stream} 227 @type sigs: [L{gpg.Signature}] 228 @param xml: the downloaded replacement feed document 229 @type xml: str 230 @type dry_run: bool 231 @return: True if the feed was updated 232 @rtype: bool 233 @since: 0.48""" 234 from . import trust 235 updated = self._oldest_trusted(sigs, trust.domain_from_url(feed_url)) 236 if updated is None: return False # None are trusted 237 238 self.update_feed_from_network(feed_url, xml, updated, dry_run = dry_run) 239 return True

240

241 - def update_interface_from_network(self, interface, new_xml, modified_time):

242 import warnings 243 warnings.warn("Use update_feed_from_network instead", DeprecationWarning, stacklevel = 2) 244 self.update_feed_from_network(interface.uri, new_xml, modified_time)

245

246 - def update_feed_from_network(self, feed_url, new_xml, modified_time, dry_run = False):

247 """Update a cached feed. 248 Called by L{update_feed_if_trusted} if we trust this data. 249 After a successful update, L{writer} is used to update the feed's 250 last_checked time. 251 @param feed_url: the feed being updated 252 @type feed_url: L{model.Interface} 253 @param new_xml: the downloaded replacement feed document 254 @type new_xml: str 255 @param modified_time: the timestamp of the oldest trusted signature (used as an approximation to the feed's modification time) 256 @type modified_time: long 257 @type dry_run: bool 258 @raises ReplayAttack: if modified_time is older than the currently cached time 259 @since: 0.48""" 260 logger.debug(_("Updating '%(interface)s' from network; modified at %(time)s") % 261 {'interface': feed_url, 'time': _pretty_time(modified_time)}) 262 263 self._import_new_feed(feed_url, new_xml, modified_time, dry_run) 264 265 if dry_run: return 266 267 feed = self.get_feed(feed_url) 268 269 from . import writer 270 feed.last_checked = int(time.time()) 271 writer.save_feed(feed) 272 273 logger.info(_("Updated feed cache entry for %(interface)s (modified %(time)s)"), 274 {'interface': feed.get_name(), 'time': _pretty_time(modified_time)})

275

276 - def _import_new_feed(self, feed_url, new_xml, modified_time, dry_run):

277 """Write new_xml into the cache. 278 @param feed_url: the URL for the feed being updated 279 @type feed_url: str 280 @param new_xml: the data to write 281 @type new_xml: str 282 @param modified_time: when new_xml was modified 283 @type modified_time: int 284 @type dry_run: bool 285 @raises ReplayAttack: if the new mtime is older than the current one""" 286 assert modified_time 287 assert isinstance(new_xml, bytes), repr(new_xml) 288 289 upstream_dir = basedir.save_cache_path(config_site, 'interfaces') 290 cached = os.path.join(upstream_dir, escape(feed_url)) 291 292 old_modified = None 293 if os.path.exists(cached): 294 with open(cached, 'rb') as stream: 295 old_xml = stream.read() 296 if old_xml == new_xml: 297 logger.debug(_("No change")) 298 # Update in-memory copy, in case someone else updated the disk copy 299 self.get_feed(feed_url, force = True) 300 return 301 old_modified = int(os.stat(cached).st_mtime) 302 303 if dry_run: 304 print(_("[dry-run] would cache feed {url} as {cached}").format( 305 url = feed_url, 306 cached = cached)) 307 from io import BytesIO 308 from zeroinstall.injector import qdom 309 root = qdom.parse(BytesIO(new_xml), filter_for_version = True) 310 feed = model.ZeroInstallFeed(root) 311 reader.update_user_feed_overrides(feed) 312 self._feeds[feed_url] = feed 313 return 314 315 # Do we need to write this temporary file now? 316 try: 317 with open(cached + '.new', 'wb') as stream: 318 stream.write(new_xml) 319 os.utime(cached + '.new', (modified_time, modified_time)) 320 new_mtime = reader.check_readable(feed_url, cached + '.new') 321 assert new_mtime == modified_time 322 323 old_modified = self._get_signature_date(feed_url) or old_modified 324 325 if old_modified: 326 if new_mtime < old_modified: 327 raise ReplayAttack(_("New feed's modification time is " 328 "before old version!\nInterface: %(iface)s\nOld time: %(old_time)s\nNew time: %(new_time)s\n" 329 "Refusing update.") 330 % {'iface': feed_url, 'old_time': _pretty_time(old_modified), 'new_time': _pretty_time(new_mtime)}) 331 if new_mtime == old_modified: 332 # You used to have to update the modification time manually. 333 # Now it comes from the signature, this check isn't useful 334 # and often causes problems when the stored format changes 335 # (e.g., when we stopped writing last-modified attributes) 336 pass 337 #raise SafeException("Interface has changed, but modification time " 338 # "hasn't! Refusing update.") 339 except: 340 os.unlink(cached + '.new') 341 raise 342 343 portable_rename(cached + '.new', cached) 344 logger.debug(_("Saved as %s") % cached) 345 346 self.get_feed(feed_url, force = True)

347

348 - def get_feed(self, url, force = False, selections_ok = False):

349 """Get a feed from the cache. 350 @param url: the URL of the feed 351 @type url: str 352 @param force: load the file from disk again 353 @type force: bool 354 @param selections_ok: if url is a local selections file, return that instead 355 @type selections_ok: bool 356 @return: the feed, or None if it isn't cached 357 @rtype: L{model.ZeroInstallFeed}""" 358 if not force: 359 feed = self._feeds.get(url, False) 360 if feed != False: 361 return feed 362 363 if url.startswith('distribution:'): 364 master_feed = self.get_feed(url.split(':', 1)[1]) 365 if not master_feed: 366 return None # Can't happen? 367 feed = self.distro.get_feed(master_feed) 368 else: 369 feed = reader.load_feed_from_cache(url, selections_ok = selections_ok) 370 if selections_ok and feed and not isinstance(feed, model.ZeroInstallFeed): 371 assert feed.selections is not None 372 return feed # (it's actually a selections document) 373 if feed: 374 reader.update_user_feed_overrides(feed) 375 self._feeds[url] = feed 376 return feed

377

378 - def get_interface(self, uri):

379 """Get the interface for uri, creating a new one if required. 380 New interfaces are initialised from the disk cache, but not from 381 the network. 382 @param uri: the URI of the interface to find 383 @type uri: str 384 @rtype: L{model.Interface}""" 385 if type(uri) == str: 386 uri = unicode(uri) 387 assert isinstance(uri, unicode) 388 389 if uri in self._interfaces: 390 return self._interfaces[uri] 391 392 logger.debug(_("Initialising new interface object for %s"), uri) 393 self._interfaces[uri] = Interface(uri) 394 reader.update_from_cache(self._interfaces[uri], iface_cache = self) 395 return self._interfaces[uri]

396

397 - def list_all_interfaces(self):

398 """List all interfaces in the cache. 399 @rtype: [str]""" 400 all = set() 401 for d in basedir.load_cache_paths(config_site, 'interfaces'): 402 for leaf in os.listdir(d): 403 if not leaf.startswith('.'): 404 all.add(unescape(leaf)) 405 return list(all) # Why not just return the set?

406

407 - def get_icon_path(self, iface):

408 """Get the path of a cached icon for an interface. 409 @param iface: interface whose icon we want 410 @type iface: L{Interface} 411 @return: the path of the cached icon, or None if not cached. 412 @rtype: str""" 413 return basedir.load_first_cache(config_site, 'interface_icons', 414 escape(iface.uri))

415

416 - def get_cached_signatures(self, uri):

417 """Verify the cached interface using GPG. 418 Only new-style XML-signed interfaces retain their signatures in the cache. 419 @param uri: the feed to check 420 @type uri: str 421 @return: a list of signatures, or None 422 @rtype: [L{gpg.Signature}] or None 423 @since: 0.25""" 424 from . import gpg 425 if os.path.isabs(uri): 426 old_iface = uri 427 else: 428 old_iface = basedir.load_first_cache(config_site, 'interfaces', escape(uri)) 429 if old_iface is None: 430 return None 431 try: 432 with open(old_iface, 'rb') as stream: 433 return gpg.check_stream(stream)[1] 434 except SafeException as ex: 435 logger.info(_("No signatures (old-style interface): %s") % ex) 436 return None

437

438 - def _get_signature_date(self, uri):

439 """Read the date-stamp from the signature of the cached interface. 440 If the date-stamp is unavailable, returns None. 441 @type uri: str 442 @rtype: int""" 443 from . import trust 444 sigs = self.get_cached_signatures(uri) 445 if sigs: 446 return self._oldest_trusted(sigs, trust.domain_from_url(uri))

447

448 - def _oldest_trusted(self, sigs, domain):

449 """Return the date of the oldest trusted signature in the list, or None if there 450 are no trusted sigs in the list. 451 @type sigs: [L{zeroinstall.injector.gpg.ValidSig}] 452 @type domain: str 453 @rtype: int""" 454 trusted = [s.get_timestamp() for s in sigs if s.is_trusted(domain)] 455 if trusted: 456 return min(trusted) 457 return None

458

459 - def mark_as_checking(self, url):

460 """Touch a 'last-check-attempt' timestamp file for this feed. 461 If url is a local path, nothing happens. 462 This prevents us from repeatedly trying to download a failing feed many 463 times in a short period. 464 @type url: str""" 465 if os.path.isabs(url): 466 return 467 feeds_dir = basedir.save_cache_path(config_site, config_prog, 'last-check-attempt') 468 timestamp_path = os.path.join(feeds_dir, model._pretty_escape(url)) 469 fd = os.open(timestamp_path, os.O_WRONLY | os.O_CREAT, 0o644) 470 os.close(fd) 471 os.utime(timestamp_path, None) # In case file already exists

472

473 - def get_last_check_attempt(self, url):

474 """Return the time of the most recent update attempt for a feed. 475 @type url: str 476 @return: The time, or None if none is recorded 477 @rtype: float | None 478 @see: L{mark_as_checking}""" 479 timestamp_path = basedir.load_first_cache(config_site, config_prog, 'last-check-attempt', model._pretty_escape(url)) 480 if timestamp_path: 481 return os.stat(timestamp_path).st_mtime 482 return None

483

484 - def get_feed_imports(self, iface):

485 """Get all feeds that add to this interface. 486 This is the feeds explicitly added by the user, feeds added by the distribution, 487 and feeds imported by a <feed> in the main feed (but not recursively, at present). 488 @type iface: L{Interface} 489 @rtype: L{Feed} 490 @since: 0.48""" 491 main_feed = self.get_feed(iface.uri) 492 if main_feed: 493 return iface.extra_feeds + main_feed.feeds 494 else: 495 return iface.extra_feeds

496

497 - def get_feeds(self, iface):

498 """Get all feeds for this interface. This is a mapping from feed URLs 499 to ZeroInstallFeeds. It includes the interface's main feed, plus the 500 resolution of every feed returned by L{get_feed_imports}. Uncached 501 feeds are indicated by a value of None. 502 @type iface: L{Interface} 503 @rtype: {str: L{ZeroInstallFeed} | None} 504 @since: 0.48""" 505 main_feed = self.get_feed(iface.uri) 506 results = {iface.uri: main_feed} 507 for imp in iface.extra_feeds: 508 try: 509 results[imp.uri] = self.get_feed(imp.uri) 510 except SafeException as ex: 511 logger.warning("Failed to load feed '%s: %s", imp.uri, ex) 512 if main_feed: 513 for imp in main_feed.feeds: 514 results[imp.uri] = self.get_feed(imp.uri) 515 return results

516

517 - def get_implementations(self, iface):

518 """Return all implementations from all of iface's feeds. 519 @type iface: L{Interface} 520 @rtype: [L{Implementation}] 521 @since: 0.48""" 522 impls = [] 523 for feed in self.get_feeds(iface).values(): 524 if feed: 525 impls += feed.implementations.values() 526 return impls

527

528 - def get_feed_targets(self, feed):

529 """Return a list of Interfaces for which feed can be a feed. 530 This is used by B{0install add-feed}. 531 @param feed: the feed 532 @type feed: L{model.ZeroInstallFeed} (or, deprecated, a URL) 533 @rtype: [model.Interface] 534 @raise SafeException: If there are no known feeds. 535 @since: 0.53""" 536 537 if not isinstance(feed, model.ZeroInstallFeed): 538 # (deprecated) 539 feed = self.get_feed(feed) 540 if feed is None: 541 raise SafeException("Feed is not cached and using deprecated API") 542 543 if not feed.feed_for: 544 raise SafeException(_("Missing <feed-for> element in '%s'; " 545 "it can't be used as a feed for any other interface.") % feed.url) 546 feed_targets = feed.feed_for 547 logger.debug(_("Feed targets: %s"), feed_targets) 548 return [self.get_interface(uri) for uri in feed_targets]

549

550 - def is_stale(self, feed_url, freshness_threshold):

551 """Check whether feed needs updating, based on the configured L{config.Config.freshness}. 552 None is considered to be stale. 553 If we already tried to update the feed within FAILED_CHECK_DELAY, returns false. 554 @type feed_url: str 555 @type freshness_threshold: int 556 @return: True if feed should be updated 557 @rtype: bool 558 @since: 0.53""" 559 if isinstance(feed_url, model.ZeroInstallFeed): 560 feed_url = feed_url.url # old API 561 elif feed_url is None: 562 return True # old API 563 564 now = time.time() 565 566 feed = self.get_feed(feed_url) 567 if feed is not None: 568 if feed.local_path is not None: 569 return False # Local feeds are never stale 570 571 if feed.last_modified is not None: 572 staleness = now - (feed.last_checked or 0) 573 logger.debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0}) 574 575 if freshness_threshold <= 0 or staleness < freshness_threshold: 576 return False # Fresh enough for us 577 # else we've never had it 578 579 last_check_attempt = self.get_last_check_attempt(feed_url) 580 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY: 581 logger.debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt)) 582 return False 583 584 return True

585

586 - def usable_feeds(self, iface, arch):

587 """Generator for C{iface.feeds} that are valid for this architecture. 588 @type iface: L{model.Interface} 589 @rtype: generator 590 @see: L{arch} 591 @since: 0.53""" 592 for f in self.get_feed_imports(iface): 593 if f.os in arch.os_ranks and f.machine in arch.machine_ranks: 594 yield f 595 else: 596 logger.debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"), 597 {'feed': f, 'os': f.os, 'machine': f.machine})

Source Code for Module zeroinstall.injector.iface_cache