Home | History | Annotate | Download | only in publish
      1 #!/usr/bin/python2.4
      2 #
      3 # CDDL HEADER START
      4 #
      5 # The contents of this file are subject to the terms of the
      6 # Common Development and Distribution License (the "License").
      7 # You may not use this file except in compliance with the License.
      8 #
      9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
     10 # or http://www.opensolaris.org/os/licensing.
     11 # See the License for the specific language governing permissions
     12 # and limitations under the License.
     13 #
     14 # When distributing Covered Code, include this CDDL HEADER in each
     15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     16 # If applicable, add the following below this CDDL HEADER, with the
     17 # fields enclosed by brackets "[]" replaced with your own identifying
     18 # information: Portions Copyright [yyyy] [name of copyright owner]
     19 #
     20 # CDDL HEADER END
     21 #
     22 # Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
     23 # Use is subject to license terms.
     24 #
     25 
     26 import sys
     27 import os
     28 import traceback
     29 import getopt
     30 import urllib
     31 import tempfile
     32 import gettext
     33 import shutil
     34 
     35 import pkg.fmri
     36 import pkg.pkgtarfile as ptf
     37 import pkg.actions as actions
     38 import pkg.manifest as manifest
     39 import pkg.server.catalog as catalog
     40 import pkg.version as version
     41 
     42 from pkg.misc import versioned_urlopen, gunzip_from_stream, msg, PipeError
     43 from pkg.client import global_settings
     44 
     45 def pname():
     46         return os.path.basename(sys.argv[0])
     47 
     48 def usage(usage_error = None):
     49 
     50         if usage_error:
     51                 error(usage_error)
     52 
     53         print >> sys.stderr, _("""\
     54 Usage:
     55         %s -r [-d dir] [-n] -v varname,url -v varname,url [-v varname,url ...] variant_type  pkgname [pkgname ...]
     56 
     57         example:
     58 
     59         %s -r -d /tmp/merge -n -v sparc,http://server1 -v i386,http://server2 arch entire
     60         """ % (pname(), pname()))
     61 
     62         sys.exit(2)
     63 
     64 def error(error):
     65         """ Emit an error message prefixed by the command name """
     66 
     67         print >> sys.stderr, pname() + ": " + error
     68 
     69 def fetch_files_byhash(server_url, hashes, pkgdir):
     70         """Given a list of files named by content hash, download from
     71         server_url into pkgdir."""
     72 
     73         req_dict = { }
     74 
     75         for i, k in enumerate(hashes):
     76                 str = "File-Name-%s" % i
     77                 req_dict[str] = k
     78 
     79         req_str = urllib.urlencode(req_dict)
     80 
     81         try:
     82                 f, v = versioned_urlopen(server_url, "filelist", [0],
     83                     data = req_str)
     84         except:
     85                 error(_("Unable to download files from: %s") % server_url)
     86                 sys.exit(1)
     87 
     88         tar_stream = ptf.PkgTarFile.open(mode = "r|", fileobj = f)
     89 
     90         if not os.path.exists(pkgdir):
     91                 try:
     92                         os.makedirs(pkgdir)
     93                 except:
     94                         error(_("Unable to create directory: %s") % pkgdir)
     95                         sys.exit(1)
     96 
     97         for info in tar_stream:
     98                 gzfobj = None
     99                 try:
    100                         # Uncompress as we retrieve the files
    101                         gzfobj = tar_stream.extractfile(info)
    102                         fpath = os.path.join(pkgdir, info.name)
    103                         outfile = open(fpath, "wb")
    104                         gunzip_from_stream(gzfobj, outfile)
    105                         outfile.close()
    106                         gzfobj.close()
    107                 except:
    108                         error(_("Unable to extract file: %s") % info.name)
    109                         sys.exit(1)
    110 
    111         tar_stream.close()
    112         f.close()
    113 
    114 manifest_cache={}
    115 null_manifest = manifest.Manifest()
    116 
    117 def get_manifest(server_url, fmri):
    118         if not fmri: # no matching fmri
    119                 return null_manifest
    120 
    121         key = "%s->%s" % (server_url, fmri)
    122         if key not in manifest_cache:
    123                 manifest_cache[key] = fetch_manifest(server_url, fmri)
    124         return manifest_cache[key]
    125 
    126 def fetch_manifest(server_url, fmri):
    127         """Fetch the manifest for package-fmri 'fmri' from the server
    128         in 'server_url'... return as Manifest object."""
    129         # Request manifest from server
    130 
    131         try:
    132                 m, v = versioned_urlopen(server_url, "manifest", [0],
    133                     fmri.get_url_path())
    134         except:
    135                 error(_("Unable to download manifest %s from %s") %
    136                     (fmri.get_url_path(), server_url))
    137                 sys.exit(1)
    138 
    139         # Read from server, write to file
    140         try:
    141                 mfst_str = m.read()
    142         except:
    143                 error(_("Error occurred while reading from: %s") % server_url)
    144                 sys.exit(1)
    145 
    146         m = manifest.Manifest()
    147         m.set_content(mfst_str)
    148 
    149         return m
    150 
    151 catalog_cache = {}
    152 
    153 def get_catalog(server_url):
    154         if server_url not in catalog_cache:
    155                 catalog_cache[server_url] = fetch_catalog(server_url)
    156         return catalog_cache[server_url][0]
    157 
    158 def cleanup_catalogs():
    159         global catalog_cache
    160         for c, d in catalog_cache.values():
    161                 shutil.rmtree(d)
    162         catalog_cache = {}
    163 
    164 def fetch_catalog(server_url):
    165         """Fetch the catalog from the server_url."""
    166 
    167         # open connection for catalog
    168         try:
    169                 c, v = versioned_urlopen(server_url, "catalog", [0])
    170         except:
    171                 error(_("Unable to download catalog from: %s") % server_url)
    172                 sys.exit(1)
    173 
    174         # make a tempdir for catalog
    175         dl_dir = tempfile.mkdtemp()
    176 
    177         # call catalog.recv to pull down catalog
    178         try:
    179                 catalog.ServerCatalog.recv(c, dl_dir)
    180         except:
    181                 error(_("Error while reading from: %s") % server_url)
    182                 sys.exit(1)
    183 
    184         # close connection to server
    185         c.close()
    186 
    187         # instantiate catalog object
    188         cat = catalog.ServerCatalog(dl_dir, read_only=True)
    189 
    190         # return (catalog, tmpdir path)
    191         return cat, dl_dir
    192 
    193 catalog_dict = {}
    194 def load_catalog(server_url):
    195         c = get_catalog(server_url)
    196         d = {}
    197         for f in c.fmris():
    198                 if f.pkg_name in d:
    199                         d[f.pkg_name].append(f)
    200                 else:
    201                         d[f.pkg_name] = [f]
    202                 for k in d.keys():
    203                         d[k].sort(reverse = True)
    204         catalog_dict[server_url] = d
    205 
    206 def expand_fmri(server_url, fmri_string, constraint=version.CONSTRAINT_AUTO):
    207         """ from specified server, find matching fmri using CONSTRAINT_AUTO
    208         cache for performance.  Returns None if no matching fmri is found """
    209         if server_url not in catalog_dict:
    210                 load_catalog(server_url)
    211 
    212         fmri = pkg.fmri.PkgFmri(fmri_string, "5.11")
    213 
    214         for f in catalog_dict[server_url].get(fmri.pkg_name, []):
    215                 if not fmri.version or f.version.is_successor(fmri.version, constraint):
    216                         return f
    217         return None
    218 
    219 def get_all_pkg_names(server_url):
    220         """ return all the pkg_names in this catalog """
    221         if server_url not in catalog_dict:
    222                 load_catalog(server_url)
    223         return catalog_dict[server_url].keys()
    224 
    225 def get_dependencies(server_url, fmri_list):
    226         s = set()
    227         for f in fmri_list:
    228                 fmri = expand_fmri(server_url, f)
    229                 _get_dependencies(s, server_url, fmri)
    230         return s
    231 
    232 def _get_dependencies(s, server_url, fmri):
    233         """ recursive incorp expansion"""
    234         s.add(fmri)
    235         for a in get_manifest(server_url, fmri).gen_actions_by_type("depend"):
    236                 if a.attrs["type"] == "incorporate":
    237                         new_fmri = expand_fmri(server_url, a.attrs["fmri"])
    238                         if new_fmri and new_fmri not in s:
    239                                 _get_dependencies(s, server_url, new_fmri)
    240         return s
    241 
    242 
    243 def main_func():
    244 
    245         basedir = None
    246         newfmri = False
    247 
    248         # XXX /usr/lib/locale is OpenSolaris-specific.
    249         gettext.install("pkgmerge", "/usr/lib/locale")
    250 
    251         global_settings.client_name = "pkgmerge"
    252 
    253         try:
    254                opts, pargs = getopt.getopt(sys.argv[1:], "d:nrv:")
    255         except getopt.GetoptError, e:
    256                 usage(_("Illegal option -- %s") % e.opt)
    257 
    258         varlist = []
    259         recursive = False
    260         get_files = True
    261 
    262         for opt, arg in opts:
    263                 if opt == "-d":
    264                         basedir = arg
    265                 if opt == "-v":
    266                         varlist.append(arg)
    267                 if opt == "-r":
    268                         recursive = True
    269                 if opt == "-n":
    270                         get_files = False
    271 
    272 
    273         if len(varlist) < 2:
    274                 usage(_("at least two -v arguments needed to merge"))
    275 
    276         if not basedir:
    277                 basedir = os.getcwd()
    278 
    279         server_list = [
    280             v.split(",", 1)[1]
    281             for v in varlist
    282         ]
    283 
    284         if len(pargs) == 1:
    285                 recursive = False
    286                 overall_set = set()
    287                 for s in server_list:
    288                         for name in get_all_pkg_names(s):
    289                                 overall_set.add(name)
    290                 fmri_arguments = list(overall_set)
    291 
    292         else:
    293                 fmri_arguments = pargs[1:]
    294 
    295         if not pargs:
    296                 usage(_("you must specify a variant"))
    297 
    298         variant = "variant.%s" % pargs[0]
    299 
    300         variant_list = [
    301                 v.split(",", 1)[0]
    302                 for v in varlist
    303                 ]
    304 
    305         fmri_expansions = []
    306 
    307         if recursive:
    308                 overall_set = set()
    309                 for s in server_list:
    310                         deps = get_dependencies(s, fmri_arguments)
    311                         for d in deps:
    312                                 if d:
    313                                         q = str(d).rsplit(":", 1)[0]
    314                                         overall_set.add(q)
    315                 fmri_arguments = list(overall_set)
    316 
    317         fmri_arguments.sort()
    318         print "Processing %d packages" % len(fmri_arguments)
    319 
    320         for fmri in fmri_arguments:
    321                 try:
    322                         fmri_list = [
    323                                 expand_fmri(s, fmri)
    324                                 for s in server_list
    325                                 ]
    326                         if len(set([
    327                                    str(f).rsplit(":", 1)[0]
    328                                    for f in fmri_list
    329                                    if f
    330                                    ])) != 1:
    331                                 error("fmris at different versions: %s" % fmri_list)
    332                                 continue
    333 
    334                 except pkg.fmri.IllegalFmri:
    335                         error(_("pkgfmri error"))
    336                         return 1
    337 
    338                 for f in fmri_list:
    339                         if f:
    340                                 basename = f.get_name()
    341                                 break
    342                 else:
    343                         error("No package of name %s in specified catalogs %s; ignoring." %\
    344                                       (fmri, server_list))
    345                         continue
    346 
    347                 merge_fmris(server_list, fmri_list, variant_list, variant, basedir, basename, get_files)
    348         cleanup_catalogs()
    349 
    350         return 0
    351 
    352 def merge_fmris(server_list, fmri_list, variant_list, variant, basedir,
    353     basename, get_files):
    354 
    355         manifest_list = [
    356                 get_manifest(s, f)
    357                 for s, f in zip(server_list, fmri_list)
    358                 ]
    359 
    360         # remove variant tags and package variant metadata
    361         # from manifests since we're reassigning...
    362         # this allows merging pre-tagged packages
    363         for m in manifest_list:
    364                 for i, a in enumerate(m.actions[:]):
    365                         if variant in a.attrs:
    366                                 del a.attrs[variant]
    367                         if a.name == "set" and a.attrs["name"] == variant:
    368                                 del m.actions[i]
    369 
    370         action_lists = manifest.Manifest.comm(*tuple(manifest_list))
    371 
    372         # set fmri actions require special merge logic.
    373         set_fmris = []
    374         for l in action_lists:
    375                 for i, a in enumerate(l):
    376                         if not (a.name == "set" and
    377                             a.attrs["name"] == "pkg.fmri"):
    378                                 continue
    379 
    380                         set_fmris.append(a)
    381                         del l[i]
    382 
    383         # If set fmris are present, then only the most recent one
    384         # and add it back to the last action list.
    385         if set_fmris:
    386                 def order(a, b):
    387                         f1 = pkg.fmri.PkgFmri(a.attrs["value"], "5.11")
    388                         f2 = pkg.fmri.PkgFmri(b.attrs["value"], "5.11")
    389                         return cmp(f1, f2)
    390                 set_fmris.sort(cmp=order)
    391                 action_lists[-1].insert(0, set_fmris[-1])
    392 
    393         for a_list, v in zip(action_lists[0:-1], variant_list):
    394                 for a in a_list:
    395                         a.attrs[variant] = v
    396 
    397         # combine actions into single list
    398         allactions = reduce(lambda a,b:a + b, action_lists)
    399 
    400         # figure out which variants are actually there for this pkg
    401         actual_variant_list = [
    402                 v
    403                 for m, v in zip(manifest_list, variant_list)
    404                 if m != null_manifest
    405                 ]
    406         print "Merging %s for %s" % (basename, actual_variant_list)
    407 
    408         # add set action to document which variants are supported
    409         allactions.append(actions.fromstr("set name=%s %s" % (variant,
    410             " ".join(["value=%s" % a
    411                       for a in actual_variant_list
    412                       ]))))
    413 
    414         allactions.sort()
    415 
    416         m = manifest.Manifest()
    417         m.actions = allactions
    418 
    419         # urlquote to avoid problems w/ fmris w/ '/' character in name
    420         basedir = os.path.join(basedir, urllib.quote(basename, ""))
    421         if not os.path.exists(basedir):
    422                 os.makedirs(basedir)
    423 
    424         m_file = file(os.path.join(basedir, "manifest"), "w")
    425         m_file.write(m.tostr_unsorted())
    426         m_file.close()
    427 
    428         for f in fmri_list:
    429                 if f:
    430                         fmri = str(f).rsplit(":", 1)[0]
    431                         break
    432         f_file = file(os.path.join(basedir, "fmri"), "w")
    433         f_file.write(fmri)
    434         f_file.close()
    435 
    436 
    437         if get_files:
    438                 # generate list of hashes for each server; last is commom
    439                 already_seen = {}
    440                 def repeated(a, d):
    441                         if a in d:
    442                                 return True
    443                         d[a] = 1
    444                         return False
    445 
    446                 hash_sets = [
    447                         set(
    448                                 [
    449                                  a.hash
    450                                  for a in action_list
    451                                  if hasattr(a, "hash") and not \
    452                                  repeated(a.hash, already_seen)
    453                                 ]
    454                                 )
    455                         for action_list in action_lists
    456                         ]
    457                 # remove duplicate files (save time)
    458 
    459                 for server, hash_set in zip(server_list + [server_list[0]], hash_sets):
    460                         if len(hash_set) > 0:
    461                                 fetch_files_byhash(server, hash_set, basedir)
    462 
    463         return 0
    464 
    465 
    466 if __name__ == "__main__":
    467         try:
    468                 ret = main_func()
    469         except SystemExit, e:
    470                 raise e
    471         except (PipeError, KeyboardInterrupt):
    472                 # We don't want to display any messages here to prevent
    473                 # possible further broken pipe (EPIPE) errors.
    474                 sys.exit(1)
    475         except:
    476                 traceback.print_exc()
    477                 sys.exit(99)
    478         sys.exit(ret)
    479 
    480