1 838 Bart #!/usr/bin/python2.4 2 838 Bart # 3 838 Bart # CDDL HEADER START 4 838 Bart # 5 838 Bart # The contents of this file are subject to the terms of the 6 838 Bart # Common Development and Distribution License (the "License"). 7 838 Bart # You may not use this file except in compliance with the License. 8 838 Bart # 9 838 Bart # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 838 Bart # or http://www.opensolaris.org/os/licensing. 11 838 Bart # See the License for the specific language governing permissions 12 838 Bart # and limitations under the License. 13 838 Bart # 14 838 Bart # When distributing Covered Code, include this CDDL HEADER in each 15 838 Bart # file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 838 Bart # If applicable, add the following below this CDDL HEADER, with the 17 838 Bart # fields enclosed by brackets "[]" replaced with your own identifying 18 838 Bart # information: Portions Copyright [yyyy] [name of copyright owner] 19 838 Bart # 20 838 Bart # CDDL HEADER END 21 838 Bart # 22 838 Bart # Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 838 Bart # Use is subject to license terms. 24 838 Bart # 25 838 Bart 26 838 Bart import sys 27 838 Bart import os 28 838 Bart import traceback 29 838 Bart import getopt 30 838 Bart import urllib 31 838 Bart import tempfile 32 838 Bart import gettext 33 838 Bart import shutil 34 838 Bart 35 838 Bart import pkg.fmri 36 838 Bart import pkg.pkgtarfile as ptf 37 838 Bart import pkg.actions as actions 38 838 Bart import pkg.manifest as manifest 39 1352 srw import pkg.server.catalog as catalog 40 838 Bart import pkg.version as version 41 838 Bart 42 838 Bart from pkg.misc import versioned_urlopen, gunzip_from_stream, msg, PipeError 43 838 Bart from pkg.client import global_settings 44 838 Bart 45 838 Bart def pname(): 46 838 Bart return os.path.basename(sys.argv[0]) 47 838 Bart 48 838 Bart def usage(usage_error = None): 49 838 Bart 50 838 Bart if usage_error: 51 838 Bart error(usage_error) 52 838 Bart 53 838 Bart print >> sys.stderr, _("""\ 54 838 Bart Usage: 55 838 Bart %s -r [-d dir] [-n] -v varname,url -v varname,url [-v varname,url ...] variant_type pkgname [pkgname ...] 56 838 Bart 57 1431 srw example: 58 838 Bart 59 838 Bart %s -r -d /tmp/merge -n -v sparc,http://server1 -v i386,http://server2 arch entire 60 838 Bart """ % (pname(), pname())) 61 838 Bart 62 838 Bart sys.exit(2) 63 838 Bart 64 838 Bart def error(error): 65 838 Bart """ Emit an error message prefixed by the command name """ 66 838 Bart 67 838 Bart print >> sys.stderr, pname() + ": " + error 68 838 Bart 69 838 Bart def fetch_files_byhash(server_url, hashes, pkgdir): 70 838 Bart """Given a list of files named by content hash, download from 71 838 Bart server_url into pkgdir.""" 72 838 Bart 73 838 Bart req_dict = { } 74 838 Bart 75 838 Bart for i, k in enumerate(hashes): 76 838 Bart str = "File-Name-%s" % i 77 838 Bart req_dict[str] = k 78 838 Bart 79 838 Bart req_str = urllib.urlencode(req_dict) 80 838 Bart 81 838 Bart try: 82 838 Bart f, v = versioned_urlopen(server_url, "filelist", [0], 83 838 Bart data = req_str) 84 838 Bart except: 85 838 Bart error(_("Unable to download files from: %s") % server_url) 86 838 Bart sys.exit(1) 87 838 Bart 88 838 Bart tar_stream = ptf.PkgTarFile.open(mode = "r|", fileobj = f) 89 838 Bart 90 838 Bart if not os.path.exists(pkgdir): 91 838 Bart try: 92 838 Bart os.makedirs(pkgdir) 93 838 Bart except: 94 838 Bart error(_("Unable to create directory: %s") % pkgdir) 95 838 Bart sys.exit(1) 96 838 Bart 97 838 Bart for info in tar_stream: 98 838 Bart gzfobj = None 99 838 Bart try: 100 838 Bart # Uncompress as we retrieve the files 101 838 Bart gzfobj = tar_stream.extractfile(info) 102 838 Bart fpath = os.path.join(pkgdir, info.name) 103 838 Bart outfile = open(fpath, "wb") 104 838 Bart gunzip_from_stream(gzfobj, outfile) 105 838 Bart outfile.close() 106 838 Bart gzfobj.close() 107 838 Bart except: 108 838 Bart error(_("Unable to extract file: %s") % info.name) 109 838 Bart sys.exit(1) 110 838 Bart 111 838 Bart tar_stream.close() 112 838 Bart f.close() 113 838 Bart 114 838 Bart manifest_cache={} 115 838 Bart null_manifest = manifest.Manifest() 116 838 Bart 117 838 Bart def get_manifest(server_url, fmri): 118 838 Bart if not fmri: # no matching fmri 119 838 Bart return null_manifest 120 838 Bart 121 838 Bart key = "%s->%s" % (server_url, fmri) 122 838 Bart if key not in manifest_cache: 123 838 Bart manifest_cache[key] = fetch_manifest(server_url, fmri) 124 838 Bart return manifest_cache[key] 125 838 Bart 126 838 Bart def fetch_manifest(server_url, fmri): 127 838 Bart """Fetch the manifest for package-fmri 'fmri' from the server 128 838 Bart in 'server_url'... return as Manifest object.""" 129 838 Bart # Request manifest from server 130 838 Bart 131 838 Bart try: 132 838 Bart m, v = versioned_urlopen(server_url, "manifest", [0], 133 838 Bart fmri.get_url_path()) 134 838 Bart except: 135 838 Bart error(_("Unable to download manifest %s from %s") % 136 838 Bart (fmri.get_url_path(), server_url)) 137 838 Bart sys.exit(1) 138 838 Bart 139 838 Bart # Read from server, write to file 140 838 Bart try: 141 838 Bart mfst_str = m.read() 142 838 Bart except: 143 838 Bart error(_("Error occurred while reading from: %s") % server_url) 144 838 Bart sys.exit(1) 145 838 Bart 146 838 Bart m = manifest.Manifest() 147 838 Bart m.set_content(mfst_str) 148 838 Bart 149 838 Bart return m 150 838 Bart 151 838 Bart catalog_cache = {} 152 838 Bart 153 838 Bart def get_catalog(server_url): 154 838 Bart if server_url not in catalog_cache: 155 838 Bart catalog_cache[server_url] = fetch_catalog(server_url) 156 838 Bart return catalog_cache[server_url][0] 157 838 Bart 158 838 Bart def cleanup_catalogs(): 159 838 Bart global catalog_cache 160 838 Bart for c, d in catalog_cache.values(): 161 838 Bart shutil.rmtree(d) 162 838 Bart catalog_cache = {} 163 838 Bart 164 838 Bart def fetch_catalog(server_url): 165 838 Bart """Fetch the catalog from the server_url.""" 166 838 Bart 167 838 Bart # open connection for catalog 168 838 Bart try: 169 838 Bart c, v = versioned_urlopen(server_url, "catalog", [0]) 170 838 Bart except: 171 838 Bart error(_("Unable to download catalog from: %s") % server_url) 172 838 Bart sys.exit(1) 173 838 Bart 174 838 Bart # make a tempdir for catalog 175 838 Bart dl_dir = tempfile.mkdtemp() 176 838 Bart 177 838 Bart # call catalog.recv to pull down catalog 178 838 Bart try: 179 1352 srw catalog.ServerCatalog.recv(c, dl_dir) 180 1431 srw except: 181 838 Bart error(_("Error while reading from: %s") % server_url) 182 838 Bart sys.exit(1) 183 838 Bart 184 838 Bart # close connection to server 185 838 Bart c.close() 186 838 Bart 187 838 Bart # instantiate catalog object 188 1352 srw cat = catalog.ServerCatalog(dl_dir, read_only=True) 189 1431 srw 190 838 Bart # return (catalog, tmpdir path) 191 838 Bart return cat, dl_dir 192 838 Bart 193 838 Bart catalog_dict = {} 194 848 Bart def load_catalog(server_url): 195 848 Bart c = get_catalog(server_url) 196 848 Bart d = {} 197 848 Bart for f in c.fmris(): 198 848 Bart if f.pkg_name in d: 199 848 Bart d[f.pkg_name].append(f) 200 848 Bart else: 201 848 Bart d[f.pkg_name] = [f] 202 848 Bart for k in d.keys(): 203 848 Bart d[k].sort(reverse = True) 204 1431 srw catalog_dict[server_url] = d 205 838 Bart 206 838 Bart def expand_fmri(server_url, fmri_string, constraint=version.CONSTRAINT_AUTO): 207 838 Bart """ from specified server, find matching fmri using CONSTRAINT_AUTO 208 838 Bart cache for performance. Returns None if no matching fmri is found """ 209 838 Bart if server_url not in catalog_dict: 210 848 Bart load_catalog(server_url) 211 838 Bart 212 1431 srw fmri = pkg.fmri.PkgFmri(fmri_string, "5.11") 213 838 Bart 214 838 Bart for f in catalog_dict[server_url].get(fmri.pkg_name, []): 215 838 Bart if not fmri.version or f.version.is_successor(fmri.version, constraint): 216 838 Bart return f 217 838 Bart return None 218 848 Bart 219 848 Bart def get_all_pkg_names(server_url): 220 848 Bart """ return all the pkg_names in this catalog """ 221 848 Bart if server_url not in catalog_dict: 222 848 Bart load_catalog(server_url) 223 848 Bart return catalog_dict[server_url].keys() 224 838 Bart 225 838 Bart def get_dependencies(server_url, fmri_list): 226 838 Bart s = set() 227 838 Bart for f in fmri_list: 228 838 Bart fmri = expand_fmri(server_url, f) 229 838 Bart _get_dependencies(s, server_url, fmri) 230 838 Bart return s 231 838 Bart 232 838 Bart def _get_dependencies(s, server_url, fmri): 233 838 Bart """ recursive incorp expansion""" 234 838 Bart s.add(fmri) 235 838 Bart for a in get_manifest(server_url, fmri).gen_actions_by_type("depend"): 236 838 Bart if a.attrs["type"] == "incorporate": 237 838 Bart new_fmri = expand_fmri(server_url, a.attrs["fmri"]) 238 838 Bart if new_fmri and new_fmri not in s: 239 838 Bart _get_dependencies(s, server_url, new_fmri) 240 838 Bart return s 241 838 Bart 242 1431 srw 243 838 Bart def main_func(): 244 838 Bart 245 838 Bart basedir = None 246 838 Bart newfmri = False 247 838 Bart 248 838 Bart # XXX /usr/lib/locale is OpenSolaris-specific. 249 838 Bart gettext.install("pkgmerge", "/usr/lib/locale") 250 838 Bart 251 838 Bart global_settings.client_name = "pkgmerge" 252 838 Bart 253 838 Bart try: 254 838 Bart opts, pargs = getopt.getopt(sys.argv[1:], "d:nrv:") 255 838 Bart except getopt.GetoptError, e: 256 1431 srw usage(_("Illegal option -- %s") % e.opt) 257 838 Bart 258 838 Bart varlist = [] 259 838 Bart recursive = False 260 838 Bart get_files = True 261 838 Bart 262 838 Bart for opt, arg in opts: 263 838 Bart if opt == "-d": 264 838 Bart basedir = arg 265 838 Bart if opt == "-v": 266 838 Bart varlist.append(arg) 267 838 Bart if opt == "-r": 268 838 Bart recursive = True 269 838 Bart if opt == "-n": 270 838 Bart get_files = False 271 1431 srw 272 1431 srw 273 838 Bart if len(varlist) < 2: 274 838 Bart usage(_("at least two -v arguments needed to merge")) 275 1431 srw 276 838 Bart if not basedir: 277 838 Bart basedir = os.getcwd() 278 838 Bart 279 848 Bart server_list = [ 280 1431 srw v.split(",", 1)[1] 281 1431 srw for v in varlist 282 1431 srw ] 283 1431 srw 284 848 Bart if len(pargs) == 1: 285 848 Bart recursive = False 286 848 Bart overall_set = set() 287 848 Bart for s in server_list: 288 848 Bart for name in get_all_pkg_names(s): 289 848 Bart overall_set.add(name) 290 848 Bart fmri_arguments = list(overall_set) 291 848 Bart 292 848 Bart else: 293 848 Bart fmri_arguments = pargs[1:] 294 1431 srw 295 1431 srw if not pargs: 296 1431 srw usage(_("you must specify a variant")) 297 838 Bart 298 838 Bart variant = "variant.%s" % pargs[0] 299 838 Bart 300 838 Bart variant_list = [ 301 838 Bart v.split(",", 1)[0] 302 838 Bart for v in varlist 303 838 Bart ] 304 838 Bart 305 838 Bart fmri_expansions = [] 306 838 Bart 307 838 Bart if recursive: 308 838 Bart overall_set = set() 309 838 Bart for s in server_list: 310 838 Bart deps = get_dependencies(s, fmri_arguments) 311 838 Bart for d in deps: 312 838 Bart if d: 313 838 Bart q = str(d).rsplit(":", 1)[0] 314 838 Bart overall_set.add(q) 315 838 Bart fmri_arguments = list(overall_set) 316 1431 srw 317 838 Bart fmri_arguments.sort() 318 838 Bart print "Processing %d packages" % len(fmri_arguments) 319 838 Bart 320 838 Bart for fmri in fmri_arguments: 321 838 Bart try: 322 838 Bart fmri_list = [ 323 838 Bart expand_fmri(s, fmri) 324 838 Bart for s in server_list 325 838 Bart ] 326 838 Bart if len(set([ 327 838 Bart str(f).rsplit(":", 1)[0] 328 838 Bart for f in fmri_list 329 838 Bart if f 330 838 Bart ])) != 1: 331 838 Bart error("fmris at different versions: %s" % fmri_list) 332 838 Bart continue 333 838 Bart 334 856 johansen except pkg.fmri.IllegalFmri: 335 838 Bart error(_("pkgfmri error")) 336 838 Bart return 1 337 838 Bart 338 838 Bart for f in fmri_list: 339 838 Bart if f: 340 838 Bart basename = f.get_name() 341 838 Bart break 342 838 Bart else: 343 838 Bart error("No package of name %s in specified catalogs %s; ignoring." %\ 344 838 Bart (fmri, server_list)) 345 838 Bart continue 346 1431 srw 347 838 Bart merge_fmris(server_list, fmri_list, variant_list, variant, basedir, basename, get_files) 348 838 Bart cleanup_catalogs() 349 838 Bart 350 838 Bart return 0 351 838 Bart 352 1431 srw def merge_fmris(server_list, fmri_list, variant_list, variant, basedir, 353 1431 srw basename, get_files): 354 838 Bart 355 838 Bart manifest_list = [ 356 838 Bart get_manifest(s, f) 357 838 Bart for s, f in zip(server_list, fmri_list) 358 838 Bart ] 359 838 Bart 360 1431 srw # remove variant tags and package variant metadata 361 838 Bart # from manifests since we're reassigning... 362 838 Bart # this allows merging pre-tagged packages 363 838 Bart for m in manifest_list: 364 838 Bart for i, a in enumerate(m.actions[:]): 365 838 Bart if variant in a.attrs: 366 838 Bart del a.attrs[variant] 367 838 Bart if a.name == "set" and a.attrs["name"] == variant: 368 838 Bart del m.actions[i] 369 838 Bart 370 838 Bart action_lists = manifest.Manifest.comm(*tuple(manifest_list)) 371 838 Bart 372 1431 srw # set fmri actions require special merge logic. 373 1431 srw set_fmris = [] 374 1431 srw for l in action_lists: 375 1431 srw for i, a in enumerate(l): 376 1431 srw if not (a.name == "set" and 377 1431 srw a.attrs["name"] == "pkg.fmri"): 378 1431 srw continue 379 1431 srw 380 1431 srw set_fmris.append(a) 381 1431 srw del l[i] 382 1431 srw 383 1431 srw # If set fmris are present, then only the most recent one 384 1431 srw # and add it back to the last action list. 385 1431 srw if set_fmris: 386 1431 srw def order(a, b): 387 1431 srw f1 = pkg.fmri.PkgFmri(a.attrs["value"], "5.11") 388 1431 srw f2 = pkg.fmri.PkgFmri(b.attrs["value"], "5.11") 389 1431 srw return cmp(f1, f2) 390 1431 srw set_fmris.sort(cmp=order) 391 1431 srw action_lists[-1].insert(0, set_fmris[-1]) 392 1431 srw 393 838 Bart for a_list, v in zip(action_lists[0:-1], variant_list): 394 838 Bart for a in a_list: 395 838 Bart a.attrs[variant] = v 396 838 Bart 397 838 Bart # combine actions into single list 398 838 Bart allactions = reduce(lambda a,b:a + b, action_lists) 399 1431 srw 400 838 Bart # figure out which variants are actually there for this pkg 401 838 Bart actual_variant_list = [ 402 838 Bart v 403 838 Bart for m, v in zip(manifest_list, variant_list) 404 838 Bart if m != null_manifest 405 838 Bart ] 406 838 Bart print "Merging %s for %s" % (basename, actual_variant_list) 407 838 Bart 408 838 Bart # add set action to document which variants are supported 409 838 Bart allactions.append(actions.fromstr("set name=%s %s" % (variant, 410 838 Bart " ".join(["value=%s" % a 411 838 Bart for a in actual_variant_list 412 838 Bart ])))) 413 838 Bart 414 838 Bart allactions.sort() 415 1431 srw 416 838 Bart m = manifest.Manifest() 417 1018 Bart m.actions = allactions 418 1431 srw 419 1018 Bart # urlquote to avoid problems w/ fmris w/ '/' character in name 420 1018 Bart basedir = os.path.join(basedir, urllib.quote(basename, "")) 421 838 Bart if not os.path.exists(basedir): 422 838 Bart os.makedirs(basedir) 423 1431 srw 424 838 Bart m_file = file(os.path.join(basedir, "manifest"), "w") 425 838 Bart m_file.write(m.tostr_unsorted()) 426 838 Bart m_file.close() 427 838 Bart 428 838 Bart for f in fmri_list: 429 838 Bart if f: 430 1018 Bart fmri = str(f).rsplit(":", 1)[0] 431 838 Bart break 432 838 Bart f_file = file(os.path.join(basedir, "fmri"), "w") 433 838 Bart f_file.write(fmri) 434 838 Bart f_file.close() 435 838 Bart 436 1431 srw 437 838 Bart if get_files: 438 848 Bart # generate list of hashes for each server; last is commom 439 848 Bart already_seen = {} 440 848 Bart def repeated(a, d): 441 848 Bart if a in d: 442 848 Bart return True 443 848 Bart d[a] = 1 444 848 Bart return False 445 848 Bart 446 838 Bart hash_sets = [ 447 838 Bart set( 448 838 Bart [ 449 838 Bart a.hash 450 838 Bart for a in action_list 451 848 Bart if hasattr(a, "hash") and not \ 452 848 Bart repeated(a.hash, already_seen) 453 838 Bart ] 454 838 Bart ) 455 838 Bart for action_list in action_lists 456 838 Bart ] 457 848 Bart # remove duplicate files (save time) 458 1431 srw 459 838 Bart for server, hash_set in zip(server_list + [server_list[0]], hash_sets): 460 838 Bart if len(hash_set) > 0: 461 838 Bart fetch_files_byhash(server, hash_set, basedir) 462 838 Bart 463 838 Bart return 0 464 838 Bart 465 838 Bart 466 838 Bart if __name__ == "__main__": 467 838 Bart try: 468 838 Bart ret = main_func() 469 838 Bart except SystemExit, e: 470 838 Bart raise e 471 838 Bart except (PipeError, KeyboardInterrupt): 472 838 Bart # We don't want to display any messages here to prevent 473 838 Bart # possible further broken pipe (EPIPE) errors. 474 838 Bart sys.exit(1) 475 838 Bart except: 476 838 Bart traceback.print_exc() 477 838 Bart sys.exit(99) 478 838 Bart sys.exit(ret) 479 838 Bart 480
Indexes created Tue Nov 24 13:07:06 UTC 2009
Terms of Use |
Privacy |
Trademarks |
Copyright Policy |
Site Guidelines |
Help
Your use of this web site or any of its content or software indicates your agreement to be bound by these Terms of Use.
Copyright © 1995-2008 Sun Microsystems, Inc.