Home | History | Annotate | Download | only in publish
      1 #!/usr/bin/python2.6
      2 #
      3 # CDDL HEADER START
      4 #
      5 # The contents of this file are subject to the terms of the
      6 # Common Development and Distribution License (the "License").
      7 # You may not use this file except in compliance with the License.
      8 #
      9 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
     10 # or http://www.opensolaris.org/os/licensing.
     11 # See the License for the specific language governing permissions
     12 # and limitations under the License.
     13 #
     14 # When distributing Covered Code, include this CDDL HEADER in each
     15 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
     16 # If applicable, add the following below this CDDL HEADER, with the
     17 # fields enclosed by brackets "[]" replaced with your own identifying
     18 # information: Portions Copyright [yyyy] [name of copyright owner]
     19 #
     20 # CDDL HEADER END
     21 #
     22 # Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
     23 # Use is subject to license terms.
     24 #
     25 
     26 import sys
     27 import os
     28 import traceback
     29 import getopt
     30 import urllib
     31 import tempfile
     32 import gettext
     33 import shutil
     34 import warnings
     35 
     36 import pkg.fmri
     37 import pkg.pkgtarfile as ptf
     38 import pkg.actions as actions
     39 import pkg.manifest as manifest
     40 import pkg.server.catalog as catalog
     41 import pkg.version as version
     42 
     43 from pkg.misc import versioned_urlopen, gunzip_from_stream, msg, PipeError
     44 from pkg.client import global_settings
     45 
     46 def pname():
     47         return os.path.basename(sys.argv[0])
     48 
     49 def usage(usage_error = None):
     50 
     51         if usage_error:
     52                 error(usage_error)
     53 
     54         print >> sys.stderr, _("""\
     55 Usage:
     56         %s -r [-d dir] [-n] -v varname,url -v varname,url [-v varname,url ...] variant_type  pkgname [pkgname ...]
     57 
     58         example:
     59 
     60         %s -r -d /tmp/merge -n -v sparc,http://server1 -v i386,http://server2 arch entire
     61         """ % (pname(), pname()))
     62 
     63         sys.exit(2)
     64 
     65 def error(error):
     66         """ Emit an error message prefixed by the command name """
     67 
     68         print >> sys.stderr, pname() + ": " + error
     69 
     70 def fetch_files_byhash(server_url, hashes, pkgdir):
     71         """Given a list of files named by content hash, download from
     72         server_url into pkgdir."""
     73 
     74         req_dict = { }
     75 
     76         for i, k in enumerate(hashes):
     77                 str = "File-Name-%s" % i
     78                 req_dict[str] = k
     79 
     80         req_str = urllib.urlencode(req_dict)
     81 
     82         try:
     83                 f, v = versioned_urlopen(server_url, "filelist", [0],
     84                     data = req_str)
     85         except:
     86                 error(_("Unable to download files from: %s") % server_url)
     87                 sys.exit(1)
     88 
     89         tar_stream = ptf.PkgTarFile.open(mode = "r|", fileobj = f)
     90 
     91         if not os.path.exists(pkgdir):
     92                 try:
     93                         os.makedirs(pkgdir)
     94                 except:
     95                         error(_("Unable to create directory: %s") % pkgdir)
     96                         sys.exit(1)
     97 
     98         for info in tar_stream:
     99                 gzfobj = None
    100                 try:
    101                         # Uncompress as we retrieve the files
    102                         gzfobj = tar_stream.extractfile(info)
    103                         fpath = os.path.join(pkgdir, info.name)
    104                         outfile = open(fpath, "wb")
    105                         gunzip_from_stream(gzfobj, outfile)
    106                         outfile.close()
    107                         gzfobj.close()
    108                 except:
    109                         error(_("Unable to extract file: %s") % info.name)
    110                         sys.exit(1)
    111 
    112         tar_stream.close()
    113         f.close()
    114 
    115 manifest_cache={}
    116 null_manifest = manifest.Manifest()
    117 
    118 def get_manifest(server_url, fmri):
    119         if not fmri: # no matching fmri
    120                 return null_manifest
    121 
    122         key = "%s->%s" % (server_url, fmri)
    123         if key not in manifest_cache:
    124                 manifest_cache[key] = fetch_manifest(server_url, fmri)
    125         return manifest_cache[key]
    126 
    127 def fetch_manifest(server_url, fmri):
    128         """Fetch the manifest for package-fmri 'fmri' from the server
    129         in 'server_url'... return as Manifest object."""
    130         # Request manifest from server
    131 
    132         try:
    133                 m, v = versioned_urlopen(server_url, "manifest", [0],
    134                     fmri.get_url_path())
    135         except:
    136                 error(_("Unable to download manifest %s from %s") %
    137                     (fmri.get_url_path(), server_url))
    138                 sys.exit(1)
    139 
    140         # Read from server, write to file
    141         try:
    142                 mfst_str = m.read()
    143         except:
    144                 error(_("Error occurred while reading from: %s") % server_url)
    145                 sys.exit(1)
    146 
    147         m = manifest.Manifest()
    148         m.set_content(mfst_str)
    149 
    150         return m
    151 
    152 catalog_cache = {}
    153 
    154 def get_catalog(server_url):
    155         if server_url not in catalog_cache:
    156                 catalog_cache[server_url] = fetch_catalog(server_url)
    157         return catalog_cache[server_url][0]
    158 
    159 def cleanup_catalogs():
    160         global catalog_cache
    161         for c, d in catalog_cache.values():
    162                 shutil.rmtree(d)
    163         catalog_cache = {}
    164 
    165 def fetch_catalog(server_url):
    166         """Fetch the catalog from the server_url."""
    167 
    168         # open connection for catalog
    169         try:
    170                 c, v = versioned_urlopen(server_url, "catalog", [0])
    171         except:
    172                 error(_("Unable to download catalog from: %s") % server_url)
    173                 sys.exit(1)
    174 
    175         # make a tempdir for catalog
    176         dl_dir = tempfile.mkdtemp()
    177 
    178         # call catalog.recv to pull down catalog
    179         try:
    180                 catalog.ServerCatalog.recv(c, dl_dir)
    181         except:
    182                 error(_("Error while reading from: %s") % server_url)
    183                 sys.exit(1)
    184 
    185         # close connection to server
    186         c.close()
    187 
    188         # instantiate catalog object
    189         cat = catalog.ServerCatalog(dl_dir, read_only=True)
    190 
    191         # return (catalog, tmpdir path)
    192         return cat, dl_dir
    193 
    194 catalog_dict = {}
    195 def load_catalog(server_url):
    196         c = get_catalog(server_url)
    197         d = {}
    198         for f in c.fmris():
    199                 if f.pkg_name in d:
    200                         d[f.pkg_name].append(f)
    201                 else:
    202                         d[f.pkg_name] = [f]
    203                 for k in d.keys():
    204                         d[k].sort(reverse = True)
    205         catalog_dict[server_url] = d
    206 
    207 def expand_fmri(server_url, fmri_string, constraint=version.CONSTRAINT_AUTO):
    208         """ from specified server, find matching fmri using CONSTRAINT_AUTO
    209         cache for performance.  Returns None if no matching fmri is found """
    210         if server_url not in catalog_dict:
    211                 load_catalog(server_url)
    212 
    213         fmri = pkg.fmri.PkgFmri(fmri_string, "5.11")
    214 
    215         for f in catalog_dict[server_url].get(fmri.pkg_name, []):
    216                 if not fmri.version or f.version.is_successor(fmri.version, constraint):
    217                         return f
    218         return None
    219 
    220 def get_all_pkg_names(server_url):
    221         """ return all the pkg_names in this catalog """
    222         if server_url not in catalog_dict:
    223                 load_catalog(server_url)
    224         return catalog_dict[server_url].keys()
    225 
    226 def get_dependencies(server_url, fmri_list):
    227         s = set()
    228         for f in fmri_list:
    229                 fmri = expand_fmri(server_url, f)
    230                 _get_dependencies(s, server_url, fmri)
    231         return s
    232 
    233 def _get_dependencies(s, server_url, fmri):
    234         """ recursive incorp expansion"""
    235         s.add(fmri)
    236         for a in get_manifest(server_url, fmri).gen_actions_by_type("depend"):
    237                 if a.attrs["type"] == "incorporate":
    238                         new_fmri = expand_fmri(server_url, a.attrs["fmri"])
    239                         if new_fmri and new_fmri not in s:
    240                                 _get_dependencies(s, server_url, new_fmri)
    241         return s
    242 
    243 
    244 def main_func():
    245 
    246         basedir = None
    247         newfmri = False
    248 
    249         # XXX /usr/lib/locale is OpenSolaris-specific.
    250         gettext.install("pkgmerge", "/usr/lib/locale")
    251 
    252         global_settings.client_name = "pkgmerge"
    253 
    254         try:
    255                opts, pargs = getopt.getopt(sys.argv[1:], "d:nrv:")
    256         except getopt.GetoptError, e:
    257                 usage(_("Illegal option -- %s") % e.opt)
    258 
    259         varlist = []
    260         recursive = False
    261         get_files = True
    262 
    263         for opt, arg in opts:
    264                 if opt == "-d":
    265                         basedir = arg
    266                 if opt == "-v":
    267                         varlist.append(arg)
    268                 if opt == "-r":
    269                         recursive = True
    270                 if opt == "-n":
    271                         get_files = False
    272 
    273 
    274         if len(varlist) < 2:
    275                 usage(_("at least two -v arguments needed to merge"))
    276 
    277         if not basedir:
    278                 basedir = os.getcwd()
    279 
    280         server_list = [
    281             v.split(",", 1)[1]
    282             for v in varlist
    283         ]
    284 
    285         if len(pargs) == 1:
    286                 recursive = False
    287                 overall_set = set()
    288                 for s in server_list:
    289                         for name in get_all_pkg_names(s):
    290                                 overall_set.add(name)
    291                 fmri_arguments = list(overall_set)
    292 
    293         else:
    294                 fmri_arguments = pargs[1:]
    295 
    296         if not pargs:
    297                 usage(_("you must specify a variant"))
    298 
    299         variant = "variant.%s" % pargs[0]
    300 
    301         variant_list = [
    302                 v.split(",", 1)[0]
    303                 for v in varlist
    304                 ]
    305 
    306         fmri_expansions = []
    307 
    308         if recursive:
    309                 overall_set = set()
    310                 for s in server_list:
    311                         deps = get_dependencies(s, fmri_arguments)
    312                         for d in deps:
    313                                 if d:
    314                                         q = str(d).rsplit(":", 1)[0]
    315                                         overall_set.add(q)
    316                 fmri_arguments = list(overall_set)
    317 
    318         fmri_arguments.sort()
    319         print "Processing %d packages" % len(fmri_arguments)
    320 
    321         for fmri in fmri_arguments:
    322                 try:
    323                         fmri_list = [
    324                                 expand_fmri(s, fmri)
    325                                 for s in server_list
    326                                 ]
    327                         if len(set([
    328                                    str(f).rsplit(":", 1)[0]
    329                                    for f in fmri_list
    330                                    if f
    331                                    ])) != 1:
    332                                 error("fmris at different versions: %s" % fmri_list)
    333                                 continue
    334 
    335                 except pkg.fmri.IllegalFmri:
    336                         error(_("pkgfmri error"))
    337                         return 1
    338 
    339                 for f in fmri_list:
    340                         if f:
    341                                 basename = f.get_name()
    342                                 break
    343                 else:
    344                         error("No package of name %s in specified catalogs %s; ignoring." %\
    345                                       (fmri, server_list))
    346                         continue
    347 
    348                 merge_fmris(server_list, fmri_list, variant_list, variant, basedir, basename, get_files)
    349         cleanup_catalogs()
    350 
    351         return 0
    352 
    353 def merge_fmris(server_list, fmri_list, variant_list, variant, basedir,
    354     basename, get_files):
    355 
    356         manifest_list = [
    357                 get_manifest(s, f)
    358                 for s, f in zip(server_list, fmri_list)
    359                 ]
    360 
    361         # remove variant tags and package variant metadata
    362         # from manifests since we're reassigning...
    363         # this allows merging pre-tagged packages
    364         for m in manifest_list:
    365                 for i, a in enumerate(m.actions[:]):
    366                         if variant in a.attrs:
    367                                 del a.attrs[variant]
    368                         if a.name == "set" and a.attrs["name"] == variant:
    369                                 del m.actions[i]
    370 
    371         action_lists = manifest.Manifest.comm(*tuple(manifest_list))
    372 
    373         # set fmri actions require special merge logic.
    374         set_fmris = []
    375         for l in action_lists:
    376                 for i, a in enumerate(l):
    377                         if not (a.name == "set" and
    378                             a.attrs["name"] == "pkg.fmri"):
    379                                 continue
    380 
    381                         set_fmris.append(a)
    382                         del l[i]
    383 
    384         # If set fmris are present, then only the most recent one
    385         # and add it back to the last action list.
    386         if set_fmris:
    387                 def order(a, b):
    388                         f1 = pkg.fmri.PkgFmri(a.attrs["value"], "5.11")
    389                         f2 = pkg.fmri.PkgFmri(b.attrs["value"], "5.11")
    390                         return cmp(f1, f2)
    391                 set_fmris.sort(cmp=order)
    392                 action_lists[-1].insert(0, set_fmris[-1])
    393 
    394         for a_list, v in zip(action_lists[0:-1], variant_list):
    395                 for a in a_list:
    396                         a.attrs[variant] = v
    397 
    398         # combine actions into single list
    399         allactions = reduce(lambda a,b:a + b, action_lists)
    400 
    401         # figure out which variants are actually there for this pkg
    402         actual_variant_list = [
    403                 v
    404                 for m, v in zip(manifest_list, variant_list)
    405                 if m != null_manifest
    406                 ]
    407         print "Merging %s for %s" % (basename, actual_variant_list)
    408 
    409         # add set action to document which variants are supported
    410         allactions.append(actions.fromstr("set name=%s %s" % (variant,
    411             " ".join(["value=%s" % a
    412                       for a in actual_variant_list
    413                       ]))))
    414 
    415         allactions.sort()
    416 
    417         m = manifest.Manifest()
    418         m.actions = allactions
    419 
    420         # urlquote to avoid problems w/ fmris w/ '/' character in name
    421         basedir = os.path.join(basedir, urllib.quote(basename, ""))
    422         if not os.path.exists(basedir):
    423                 os.makedirs(basedir)
    424 
    425         m_file = file(os.path.join(basedir, "manifest"), "w")
    426         m_file.write(m.tostr_unsorted())
    427         m_file.close()
    428 
    429         for f in fmri_list:
    430                 if f:
    431                         fmri = str(f).rsplit(":", 1)[0]
    432                         break
    433         f_file = file(os.path.join(basedir, "fmri"), "w")
    434         f_file.write(fmri)
    435         f_file.close()
    436 
    437 
    438         if get_files:
    439                 # generate list of hashes for each server; last is commom
    440                 already_seen = {}
    441                 def repeated(a, d):
    442                         if a in d:
    443                                 return True
    444                         d[a] = 1
    445                         return False
    446 
    447                 hash_sets = [
    448                         set(
    449                                 [
    450                                  a.hash
    451                                  for a in action_list
    452                                  if hasattr(a, "hash") and not \
    453                                  repeated(a.hash, already_seen)
    454                                 ]
    455                                 )
    456                         for action_list in action_lists
    457                         ]
    458                 # remove duplicate files (save time)
    459 
    460                 for server, hash_set in zip(server_list + [server_list[0]], hash_sets):
    461                         if len(hash_set) > 0:
    462                                 fetch_files_byhash(server, hash_set, basedir)
    463 
    464         return 0
    465 
    466 
    467 if __name__ == "__main__":
    468 
    469         # Make all warnings be errors.
    470         warnings.simplefilter('error')
    471 
    472         try:
    473                 ret = main_func()
    474         except SystemExit, e:
    475                 raise e
    476         except (PipeError, KeyboardInterrupt):
    477                 # We don't want to display any messages here to prevent
    478                 # possible further broken pipe (EPIPE) errors.
    479                 sys.exit(1)
    480         except:
    481                 traceback.print_exc()
    482                 sys.exit(99)
    483         sys.exit(ret)
    484 
    485