This script creates .files.tar.gz for Arch Linux repositories for pkgtools.
This file will be used by pkgfile of the pkgtools package to provide a list of files which will be installed by the package.

Just change the config variable to your needs and start the scripts with the wanted repository and architecture or all as parameter.
E.g.:
createfiles.py --repo=„i686,x86_64“
or
createfiles.py --all

Dependencies:

• python
• xz

Version History:

0.2.1:

• fixed file repo.db is skipped

0.2:

• support for xz-packages
• speed improvement through less extracting calls

0.1:

• creates .files.tar.gz archives for each given architecture in each given repository
• creates a backup of old .files.tar.gz
• one varialbe configures whole script

createfiles.py

#!/usr/bin/env python
 
######################################################################
#
#                  Author: Andrwe Lord Weber
#                  E-Mail: lord-weber-andrwe<at>renona-studios<dot>org
#                  Version: 0.2.1 
#                  Description:
#                       This script builds files.tar.gz archives
#                       for repositories of Arch Linux
#
######################################################################
 
import sys, os, getopt, subprocess,  atexit, tarfile, re, shutil, subprocess
from signal import signal, SIGTERM, SIGINT
from random import random
 
 
# hash array for setting up needed values
# path should be without / at the end
# structure:
#			"repository":	{
#									"architecture": "/repository/path",
#									"architecture2": "/repository/path2"
#								},
#     "repository2": {
#               }
config = {
				"andrwe": {
									"i686": "/data/repo/pkgs/i686",
									"x86_64": "/data/repo/pkgs/x86_64",
								}
			}
 
# static variables
# extension of database archive of the repositories
DBEXT = ".db.tar.gz"
# extension of files archive of the repositories
FILEXT = ".files.tar.gz"
# directory for temporary files
TMPDIR = "/tmp/createfiles.py." + str(random())
# binary of xz-utils, if None it is searched in $PATH
XZBIN = None
 
 
######################################################################
#
#                  After this line no changes are needed
#
######################################################################
 
def usage():
	print ""
	print ""
	print "This is create_files.py (C) 2010 by Andrwe Lord Weber <lord-weber-andrwe AT renona-studios DOT org>"
	print ""
	print "Usage:"
	print "  create_files.py --<repository=\"architecture1,architecture2\"|all>"
	print ""
	print "Configured repositories with architectures are:"
	for repo,archs in config.iteritems():
		for arch,path in archs.iteritems():
			print "  --" + str(repo) + "=" + str(arch) + " with path: " + str(path)
	sys.exit(0)
 
 
# check whether program path is correct
def _progpath(_program):
	"""Get Full path for given Program"""
	for _path in os.environ.get('PATH', '').split(':'):
		if os.path.exists(os.path.join(_path, _program)) and not os.path.isdir(os.path.join(_path, _program)):
			return os.path.join(_path, _program)
	return None
 
 
# return path of program
def _get_progbin(_prog, _var):
	if not _var == None and os.path.exists(_var) and not os.path.isdir(_var):
		return _var
	else:
		return _progpath(_prog)
 
 
# open tar-file and return tar-object
def _get_tarfile_obj(_pkgfile):
	_filelen = len(_pkgfile)
 
	if _pkgfile[_filelen-2:] == "xz":
		_xzfile = _pkgfile
		_pkgfile = TMPDIR + "/" + _pkgfile[:_filelen-3].split("/")[-1]
		if not os.path.exists(_pkgfile):
			_blaufile = open(_pkgfile, 'w')
			try:
				_xzbin = [_get_progbin("xz", XZBIN), "-dc", _xzfile]
				_suberr = subprocess.Popen(_xzbin, stdout=_blaufile, stderr=subprocess.PIPE).communicate()
			except:
				print sys.exc_value
				sys.exit(1)
			_blaufile.close()
 
	try:
		tarfile.is_tarfile(_pkgfile)
	except IOError:
		print "Error: The file " + _pkgfile + " doesn't exist or isn't readable."
		sys.exit(1)
	except:
		print "Error: An unexpected error occured while running tarfile.is_tarfile in _get_pkginfo() with parameters: " + _pkgfile + " " + _value
		sys.exit(1)
 
	try:
		_file = tarfile.open(_pkgfile)
	except CompressionError:
		print "Error: The compression of " + _pkgfile + " is not supported."
		sys.exit(1)
	except:
		print "Error: An unknown error occured while running tarfile.open in _get_pkginfo() with parameters: " + _pkgfile + " " + _value
		sys.exit(1)
	return _file
 
 
# extract content of .PKGINFO file of tar-object
def _get_pkginfo(_file):
	try:
		_pkginfo_file = _file.extractfile(".PKGINFO").read() 
	except:
		print "Error: An unknown error occured while extracting .PKGINFO in _get_pkginfo() with parameters: " + _pkgfile + " " + _value
		sys.exit(1)
 
	return _pkginfo_file
 
 
# get value of variables in .PKGINFO e.g. pkgname
def _get_pkgvalue(pkgfile, value):
	try:
		reg = re.compile(value, re.M)
		match = reg.search(pkgfile)
		return match.group(1)
	except:
		print "Error: An unknown error occured while running re.compile() in _get_pkginfo() with parameters: " + pkgfile + " " + value
		sys.exit(1)
 
 
# get filenames of tar-object	
def _get_pkgfiles(_file):
	try:
		_files = _file.getnames() 
	except CompressionError:
		print "Error: The compression of " + pkgfile + " is not supported."
		sys.exit(1)
	except:
		print "Error: An unknown error occured while running tarfile.open in _get_pkgfiles() with parameters: " + pkgfile
		sys.exit(1)
 
	try:
		_reg = re.compile("^(\..*)$")
		_newfiles = []
		for _tmpfile in _files:			
			if not _reg.search(_tmpfile):
				_newfiles.append(_tmpfile + "\n")
		del _files
		return _newfiles
	except:
		print "Error: An unknown error occured while running re.compile() in _get_pkginfo() with parameters: " + _file
		sys.exit(1)
 
 
# builds 'files' file for pkgfile for given repository and architecture
def _build_pkgfile(_filepath, repo, arch):
	_content = ["%FILES%\n"]
	_tarfile_obj = _get_tarfile_obj(_filepath)
	_pkgfile = _get_pkginfo(_tarfile_obj)
	_pkgname = _get_pkgvalue(_pkgfile, "^pkgname = (.*)$")
	_pkgver = _get_pkgvalue(_pkgfile, "^pkgver = (.*)$")
	_repodir = repo + "/" + arch
	_tmppkgdir = TMPDIR + "/" + _repodir + "/" + _pkgname + "-" + _pkgver
	_pkgfiles = _get_pkgfiles(_tarfile_obj)
	_content.extend(_pkgfiles)
	_tarfile_obj.close()
	try:
		os.makedirs(_tmppkgdir)
		try:
			_pkgfile = open(_tmppkgdir + "/files", 'w')
			_pkgfile.writelines(_content)
			_pkgfile.close()
		except:
			print sys.exc_info()
			print "Error: Couldn't create file " + _tmppkgdir + "/files"
			sys.exit(1)
	except:
		print "Error: Couldn't create folder " + _tmppkgdir
		sys.exit(1)
 
 
# remove temporary directory after sys.exit()
def cleanup():
	try:
		os.rmdir(TMPDIR)
	finally:
		sys.exit(0)
 
 
 
def main(argv):
	_repos = ["all"]
	_all = 0
 
	# compile argument list
	for repo, archs in config.iteritems():
		_repos.append(repo + "=")
	try:
		opts, args = getopt.getopt(argv, "", _repos)
	except getopt.GetoptError, err:
		print str(err)
		usage()
 
	# check for --all parameter in all given arguments
	for opt in opts:
		if opt.count("--all") >= 1:
			_all = 1
			print "Option '--all' found. Package will be build for all repositories."
 
	os.mkdir(TMPDIR)
	if _all == 1: # build files.tar. for each configured repository
		for repo,archs in config.iteritems():
			for arch,path in archs.iteritems():
				print "Building " + FILEXT + " for " + repo + "/" + arch
				try:
					path = path + "/"
					pkgfiles = repo + FILEXT
					repofiles = repo + DBEXT					
					for file in os.listdir(path):
						try:
							filepath = path +  file
							if filepath != path + repofiles and filepath != path + repofiles + ".old" and filepath != path + pkgfiles and filepath != path + pkgfiles + ".old" and filepath != path + repo + ".db":
								_build_pkgfile(filepath, repo[2:], arch)
						except KeyError:
							print "Error: The given architecture " + arch + " doesn't exist."
							sys.exit(1)
						except:
							print "Error: An error occured in main->for repo, arg in opts:. Path: " + filepath
							sys.exit(1)
					os.chdir(TMPDIR + "/" + repo[2:] + "/" + arch)					
					filesarch = tarfile.open(pkgfiles, 'w:gz')
					for dir in os.listdir("./"):
						if not dir == pkgfiles:
							filesarch.add(dir)
					filesarch.close()
					if os.path.isfile(path + pkgfiles):
						try:				
							if tarfile.is_tarfile(path + pkgfiles):
								try:
									shutil.move(path + pkgfiles, path + pkgfiles + ".old")
								except:
									print "Error: Couldn't move " + path + pkgfiles + " to " + path + pkgfiles + ".old"
									sys.exit(1)
						except:
							print "Error: Existing file " + path + pkgfiles + " isn't a valid tar archive."
							sys.exit(1)
					if os.path.isfile(pkgfiles):
						try:				
							if tarfile.is_tarfile(pkgfiles):
								try:
									shutil.move(pkgfiles, path + pkgfiles)
								except:
									print "Error: Couldn't move " + pkgfiles + " which was build by this script to " + path + pkgfiles
									sys.exit(1)
						except:
							print "Error: The archive file " + pkgfiles + " build by this script isn't a valid tar archive."
							sys.exit(1)
				except:
					print "Error: An error occured in main->for repo, arg in opts:"
	else: # builds files.tar. for each given repository
		for repo, arg in opts:
			for arch in arg.split(","):
				print "Building " + FILEXT + " for " + repo[2:] + "/" + arch
				path = config[repo[2:]][arch] + "/"
				pkgfiles = repo[2:] + FILEXT
				repofiles = repo[2:] + DBEXT
				for file in os.listdir(path):
					try:
						filepath = path + file
						if filepath != path + repofiles and filepath != path + repofiles + ".old" and filepath != path + pkgfiles and filepath != path + pkgfiles + ".old" and filepath != path + repo[2:] + ".db":
							_build_pkgfile(filepath, repo[2:], arch)
					except KeyError:
						print "Error: The given architecture " + arch + " doesn't exist."
						sys.exit(1)
					except:
						print "Error: An error occured in main->'for repo, arg in opts:' filepath:" + filepath
						sys.exit(1)
				os.chdir(TMPDIR + "/" + repo[2:] + "/" + arch)
				filesarch = tarfile.open(pkgfiles, 'w:gz')
				for dir in os.listdir("./"):
					if not dir == pkgfiles:
						filesarch.add(dir)
				filesarch.close()
				if os.path.isfile(path + pkgfiles):
					try:				
						if tarfile.is_tarfile(path + pkgfiles):
							try:
								shutil.move(path + pkgfiles, path + pkgfiles + ".old")
							except:
								print "Error: Couldn't move " + path + pkgfiles + " to " + path + pkgfiles + ".old"
								sys.exit(1)
					except:
						print "Error: Existing file " + path + pkgfiles + " isn't a valid tar archive."
						sys.exit(1)
				if os.path.isfile(pkgfiles):
					try:				
						if tarfile.is_tarfile(pkgfiles):
							try:
								shutil.move(pkgfiles, path + pkgfiles)
							except:
								print "Error: Couldn't move " + pkgfiles + " which was build by this script to " + path + pkgfiles
								sys.exit(1)
					except:
						print "Error: The archive file " + pkgfiles + " build by this script isn't a valid tar archive."
						sys.exit(1)
	shutil.rmtree(TMPDIR)
 
# Start main-function
 
if __name__ == "__main__":
	atexit.register(cleanup)
	signal(SIGTERM, lambda signum, stack_frame: exit(1))
	signal(SIGINT, lambda signum, stack_frame: exit(1))
 
	if len(sys.argv) > 1:
		main(sys.argv[1:])
	else:
		print "Error: You have to give at least one architecture or '--all' as parameter!!"
		usage()