#!/usr/bin/python -tt
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2010 Red Hat, Inc
# Written By Seth Vidal - skvidal@fedoraproject.org
# take a localpath, a remote (read) path and a remote (write) path
# take the pkgs in the localpath
# the repodata in the remote read path
# add the pkgs in the localpath to the repodata from the remote read path
# write the resulting repodata out to the remote (write) path
# grab remote repomd.xml and other metadata
# createrepo job with local pkgs + "remote" repos - but fake out the url paths
# for the remote pkgs
# grab all non primary/file/other data (or databases) from the repomd.xml and
# refer to them as well
import sys
import os
import time
import createrepo
import yum
from yum.misc import to_xml
import rpmUtils.arch
import shutil
from optparse import OptionParser
import getpass
import paramiko
def parse_ssh_string(hoststring):
# [username@]hostname[:port]/path/to/files
username = getpass.getuser()
port = 22
user_idx = hoststring.find('@')
if user_idx != -1:
username = hoststring[:user_idx]
user_idx += 1 # get rid of the @
else:
user_idx = 0
path_idx = hoststring.find('/')
filepath = hoststring[path_idx:]
hostspec = hoststring[user_idx:path_idx]
if hoststring.find(':') != -1:
hostname, port = hostspec.split(':')
else:
hostname = hostspec
return username, hostname, port, filepath
def setup_sftp_connection(hoststring):
username, hostname, port, filepath = parse_ssh_string(hoststring)
s_client = paramiko.SSHClient()
#s_client
pass
#override how we dump the remote repo out
#FIXME - need to make sure the remote pkg in this case is not a mergerepo'd
# pkg and therefore has an absolute url that actually matters :(
def _local_yap_return_rem_loc(self):
return """\n""" % to_xml(self.relativepath, attrib=True)
yum.packages.YumAvailablePackage._return_remote_location = _local_yap_return_rem_loc
# end the override
class RepoDataObject(object):
"""basic datatype object for repomd.xml mdtypes"""
def __init__(self):
self._special_cases = ['location', 'checksum', 'open-checksum']
self.type = ''
self.info_dict = {'location': (None, ''),
'timestamp': 0,
'size': 0,
'open-size': 0,
'checksum' : ('',''),
'open-checksum': ('',''),
'database_version': None}
def dump_xml(self):
msg = ""
top = """\n""" % to_xml(self.type, attrib=True)
msg += top
for data in ['checksum', 'open-checksum']:
if self.info_dict[data][0]:
d_xml = """ <%s type="%s">%s\n""" % (data,
to_xml(self.info_dict[data][0], attrib=True),
to_xml(self.info_dict[data][1]), data)
msg += d_xml
if self.info_dict['location'][1]:
loc = """ \n""" % to_xml(
self.info_dict['location'][1], attrib=True)
if self.info_dict['location'][0]:
loc = """ \n""" % (
to_xml(self.info_dict['location'][0], attrib=True),
to_xml(self.info_dict['location'][1], attrib=True))
msg += loc
for data in self.info_dict:
if data in self._special_cases:
continue
if self.info_dict[data]:
d_xml = """ <%s>%s\n""" % (data, to_xml(self.info_dict[data]),
data)
msg += d_xml
bottom = """\n"""
msg += bottom
return msg
class NewRepoXML(object):
"""basic repomd.xml representing object that allows modification and writing"""
def __init__(self):
self.data = [] # list of our repodata content objects
self.revision = str(int(time.time()))
self.content_tags = []
self.repo_tags = []
self.distro_tags = []
def dump_xml(self):
msg = ""
top = """
\n"""
msg += top
rev = """ %s\n""" % self.revision
msg += rev
if self.content_tags or self.distro_tags or self.repo_tags:
tags = """\n"""
for item in self.content_tags:
tag = """ %s\n""" % (to_xml(item))
tags += tag
for item in self.repo_tags:
tag = """ %s\n""" % (to_xml(item))
tags += tag
for (cpeid, item) in self.distro_tags:
itemlist = list(item) # frellingsets.
if cpeid:
tag = """ %s\n""" % (
to_xml(cpeid, attrib=True), to_xml(itemlist[0]))
else:
tag = """ %s\n""" % (to_xml(itemlist[0]))
tags += tag
tags += """\n"""
msg += tags
for md in self.data:
msg += md.dump_xml()
msg += """\n"""
return msg
class Syncrepo(object):
def __init__(self):
self.outputdir = os.getcwd()
self.baserepo = None
self.repo = None # yum repo object of the above
self.basedir = os.getcwd()
self.verbose = None
self.quiet = None
self.localpkgs = []
self.remote_pkgs = []
self.mdconf = createrepo.MetaDataConfig()
self.yumbase = yum.YumBase()
self.yumbase.conf.cachedir = yum.misc.getCacheDir()
self.yumbase.conf.cache = 0
# default to all arches
self.archlist = yum.misc.unique(rpmUtils.arch.arches.keys() + rpmUtils.arch.arches.values())
def _setup_createrepo(self):
self.mdconf.basedir = self.basedir
self.mdconf.verbose = self.verbose
self.mdconf.quiet = self.quiet
self.mdconf.database = True
self.mdbase_class = createrepo.MetaDataGenerator
return self.mdbase_class(config_obj=self.mdconf)
def _open_repo(self, repourl):
self.yumbase.repos.disableRepo('*')
repoid = 'remote_repo' # maybe make this predictably replaceable - but not now
repo = self.yumbase.add_enable_repo(repoid, baseurls=[repourl])
#setup our sacks
self.yumbase._getSacks(archlist=self.archlist)
thisrepo = self.yumbase.repos.findRepos(repoid)[0]
return thisrepo
def _get_remote_pkgs(self, baserepo=None):
if not baserepo:
baserepo = self.baserepo
self.repo = self._open_repo(baserepo)
self.remote_pkgs = self.repo.sack.returnPackages()
def _open_local_pkgs(self, pkgs=[]):
ret_pkgs = {}
if not pkgs:
pkgs = self.localpkgs
# return local pkg objects attached to the local pkg path
# so we can look at these vs the remote pkgs for pruning out
# any remote pkgs
for pkg in pkgs:
pkgpath = self.basedir + '/' + pkg
po = yum.packages.YumLocalPackage(self.yumbase.ts, pkgpath)
ret_pkgs[pkg] = po
return ret_pkgs
def makerepo(self):
"""creates the new repodata from the pkgs we have"""
remote_paths = {}
self._get_remote_pkgs()
for rpkg in self.remote_pkgs:
remote_paths[rpkg.relativepath] = rpkg
for pkgpath, po in self._open_local_pkgs().items():
if po.localpath in remote_paths:
if po.checksum != remote_paths[po.localpath].checksum:
del remote_paths[po.localpath]
else:
# if they are the same checksum, just skip this one locally
self.localpkgs.remove(pkgpath)
pkgs = self.localpkgs + remote_paths.values()
self.mdconf.pkglist = pkgs
self.mdconf.outputdir = self.outputdir
self.mdconf.directories = [ self.basedir ]
# clean out what was there
if os.path.exists(self.mdconf.outputdir + '/repodata'):
shutil.rmtree(self.mdconf.outputdir + '/repodata')
if not os.path.exists(self.mdconf.outputdir):
os.makedirs(self.mdconf.outputdir)
mdgen = self._setup_createrepo()
mdgen.doPkgMetadata()
mdgen.doRepoMetadata()
mdgen.doFinalMove()
if hasattr(mdgen, 'read_pkgs'):
self.read_pkgs = mdgen.read_pkgs
repomdxml = self.outputdir + '/repodata/repomd.xml'
repomd = yum.repoMDObject.RepoMD('newrepo', repomdxml)
ignore_dtypes = ['primary', 'primary_db', 'filelists', 'filelists_db',
'other', 'other_db']
new_repomd = NewRepoXML()
if 'content' in self.repo.repoXML.tags:
new_repomd.content_tags = self.repo.repoXML.tags['content']
if 'distro' in self.repo.repoXML.tags:
new_repomd.distro_tags = self.repo.repoXML.tags['distro'].items()
if 'repo' in self.repo.repoXML.tags:
new_repomd.distro_tags = self.repo.repoXML.tags['repo']
for (dtype,dobj) in self.repo.repoXML.repoData.items():
if dtype in ignore_dtypes:
continue
rd = RepoDataObject()
rd.type = dtype
rd.info_dict = {'location': dobj.location,
'timestamp': dobj.timestamp,
'size': dobj.size,
'open-size': dobj.opensize,
'checksum' : dobj.checksum,
'open-checksum': dobj.openchecksum,
'database_version': dobj.dbversion}
new_repomd.data.append(rd)
for (dtype, dobj) in repomd.repoData.items():
if dtype not in ignore_dtypes:
continue
rd = RepoDataObject()
rd.type = dtype
rd.info_dict = {'location': dobj.location,
'timestamp': dobj.timestamp,
'size': dobj.size,
'open-size': dobj.opensize,
'checksum' : dobj.checksum,
'open-checksum': dobj.openchecksum,
'database_version': dobj.dbversion}
new_repomd.data.append(rd)
fo = open(repomdxml, 'w')
fo.write(new_repomd.dump_xml())
fo.flush()
fo.close()
def parse_args(args):
"""Parse our opts/args"""
usage = """
sync_to_repo --baserepo=url --outputdir=/some/place pkg1 pkg2 pkg3 /dir/of/pkgs
"""
parser = OptionParser(version = "sync_to_repo 0.1", usage=usage)
# query options
parser.add_option("-o", "--outputdir", default=None,
help="Location to create the repository")
parser.add_option("-b", "--baserepo", default=None,
help="location to get the basic repo we want to sync to")
parser.add_option("--basedir", default=os.getcwd(),
help="base dir to look for specified pkgs from")
parser.add_option("-s", "--ssh", default=None,
help="send files to the following location via ssh")
parser.add_option("-v", "--verbose", default=False, action='store_true',
help="more output")
parser.add_option("-q", "--quiet", default=False, action='store_true',
help="less output")
(opts, argsleft) = parser.parse_args(args)
if not opts.baserepo:
parser.print_usage()
sys.exit(1)
return opts, argsleft
def main(args):
"""main"""
opts,args = parse_args(args)
syncrepo = Syncrepo()
syncrepo.outputdir = opts.outputdir
syncrepo.baserepo = opts.baserepo
syncrepo.basedir = opts.basedir
syncrepo.verbose = opts.verbose
syncrepo.quiet = opts.quiet
# FIXME - take args - take dirs and find rpms
# non-dirs - treat as rpms
# relative paths matter here.
syncrepo.localpkgs = args
syncrepo.makerepo()
if __name__ == "__main__":
main(sys.argv[1:])