forked from public-mirrors/BorgExtend
moving to its own repo
This commit is contained in:
brent s
commit
08fc183956
9 changed files with 1732 additions and 0 deletions
97
plugins/ldap.py
Normal file
97
plugins/ldap.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import os
|
||||
# TODO: virtual env?
|
||||
import ldap
|
||||
import ldif
|
||||
|
||||
|
||||
# Designed for use with OpenLDAP in an OLC configuration.
|
||||
|
||||
|
||||
class Backup(object):
|
||||
def __init__(self,
|
||||
server = 'ldap://sub.domain.tld',
|
||||
port = 389,
|
||||
basedn = 'dc=domain,dc=tld',
|
||||
sasl = False,
|
||||
starttls = True,
|
||||
binddn = 'cn=Manager,dc=domain,dc=tld',
|
||||
password_file = '~/.ldap.pass',
|
||||
password = None,
|
||||
outdir = '~/.cache/backup/ldap',
|
||||
splitldifs = True):
|
||||
self.server = server
|
||||
self.port = port
|
||||
self.basedn = basedn
|
||||
self.sasl = sasl
|
||||
self.binddn = binddn
|
||||
self.outdir = os.path.abspath(os.path.expanduser(outdir))
|
||||
os.makedirs(self.outdir, exist_ok = True)
|
||||
os.chmod(self.outdir, mode = 0o0700)
|
||||
self.splitldifs = splitldifs
|
||||
self.starttls = starttls
|
||||
if password_file and not password:
|
||||
with open(os.path.abspath(os.path.expanduser(password_file)), 'r') as f:
|
||||
self.password = f.read().strip()
|
||||
else:
|
||||
self.password = password
|
||||
# Human readability, yay.
|
||||
# A note, SSLv3 is 0x300. But StartTLS can only be done with TLS, not SSL, I *think*?
|
||||
# PRESUMABLY, now that it's finalized, TLS 1.3 will be 0x304.
|
||||
# See https://tools.ietf.org/html/rfc5246#appendix-E
|
||||
self._tlsmap = {'1.0': int(0x301), # 769
|
||||
'1.1': int(0x302), # 770
|
||||
'1.2': int(0x303)} # 771
|
||||
self._minimum_tls_ver = '1.2'
|
||||
if self.sasl:
|
||||
self.server = 'ldapi:///'
|
||||
self.cxn = None
|
||||
self.connect()
|
||||
self.dump()
|
||||
self.close()
|
||||
|
||||
def connect(self):
|
||||
self.cxn = ldap.initialize(self.server)
|
||||
self.cxn.set_option(ldap.OPT_REFERRALS, 0)
|
||||
self.cxn.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
|
||||
if not self.sasl:
|
||||
if self.starttls:
|
||||
self.cxn.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
|
||||
self.cxn.set_option(ldap.OPT_X_TLS, ldap.OPT_X_TLS_DEMAND)
|
||||
self.cxn.set_option(ldap.OPT_X_TLS_DEMAND, True)
|
||||
self.cxn.set_option(ldap.OPT_X_TLS_PROTOCOL_MIN, self._tlsmap[self._minimum_tls_ver])
|
||||
if self.sasl:
|
||||
self.cxn.sasl_external_bind_s()
|
||||
else:
|
||||
if self.starttls:
|
||||
self.cxn.start_tls_s()
|
||||
self.cxn.bind_s(self.binddn, self.password)
|
||||
return()
|
||||
|
||||
def dump(self):
|
||||
dumps = {'schema': 'cn=config',
|
||||
'data': self.basedn}
|
||||
with open(os.path.join(self.outdir, ('ldap-config.ldif' if self.splitldifs else 'ldap.ldif')), 'w') as f:
|
||||
l = ldif.LDIFWriter(f)
|
||||
rslts = self.cxn.search_s(dumps['schema'],
|
||||
ldap.SCOPE_SUBTREE,
|
||||
filterstr = '(objectClass=*)',
|
||||
attrlist = ['*', '+'])
|
||||
for r in rslts:
|
||||
l.unparse(r[0], r[1])
|
||||
if self.splitldifs:
|
||||
f = open(os.path.join(self.outdir, 'ldap-data.ldif'), 'w')
|
||||
else:
|
||||
f = open(os.path.join(self.outdir, 'ldap.ldif'), 'a')
|
||||
rslts = self.cxn.search_s(dumps['data'],
|
||||
ldap.SCOPE_SUBTREE,
|
||||
filterstr = '(objectClass=*)',
|
||||
attrlist = ['*', '+'])
|
||||
l = ldif.LDIFWriter(f)
|
||||
for r in rslts:
|
||||
l.unparse(r[0], r[1])
|
||||
f.close()
|
||||
|
||||
def close(self):
|
||||
if self.cxn:
|
||||
self.cxn.unbind_s()
|
||||
return()
|
||||
96
plugins/mysql.py
Normal file
96
plugins/mysql.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
import copy
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import warnings
|
||||
|
||||
_mysql_ssl_re = re.compile('^ssl-(.*)$')
|
||||
|
||||
# TODO: is it possible to do a pure-python dump via PyMySQL?
|
||||
# TODO: add compression support? Not *that* necessary since borg has its own.
|
||||
# in fact, it's better to not do it on the dumps directly so borg can diff/delta better.
|
||||
|
||||
class Backup(object):
|
||||
def __init__(self, dbs = None,
|
||||
cfg = '~/.my.cnf',
|
||||
cfgsuffix = '',
|
||||
splitdumps = True,
|
||||
dumpopts = None,
|
||||
mysqlbin = 'mysql',
|
||||
mysqldumpbin = 'mysqldump',
|
||||
outdir = '~/.cache/backup/mysql'):
|
||||
# If dbs is None, we dump ALL databases (that the user has access to).
|
||||
self.dbs = dbs
|
||||
self.cfgsuffix = cfgsuffix
|
||||
self.splitdumps = splitdumps
|
||||
self.mysqlbin = mysqlbin
|
||||
self.mysqldumpbin = mysqldumpbin
|
||||
self.outdir = os.path.abspath(os.path.expanduser(outdir))
|
||||
self.cfg = os.path.abspath(os.path.expanduser(cfg))
|
||||
os.makedirs(self.outdir, exist_ok = True)
|
||||
os.chmod(self.outdir, mode = 0o0700)
|
||||
if not os.path.isfile(self.cfg):
|
||||
raise OSError(('{0} does not exist!').format(self.cfg))
|
||||
if not dumpopts:
|
||||
self.dumpopts = ['--routines',
|
||||
'--add-drop-database',
|
||||
'--add-drop-table',
|
||||
'--allow-keywords',
|
||||
'--complete-insert',
|
||||
'--create-options',
|
||||
'--extended-insert']
|
||||
else:
|
||||
self.dumpopts = dumpopts
|
||||
self.getDBs()
|
||||
self.dump()
|
||||
|
||||
def getDBs(self):
|
||||
if not self.dbs:
|
||||
_out = subprocess.run([self.mysqlbin, '-BNne', 'SHOW DATABASES'],
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.PIPE)
|
||||
if _out.returncode != 0:
|
||||
raise RuntimeError(('Could not successfully list databases: '
|
||||
'{0}').format(_out.stderr.decode('utf-8')))
|
||||
self.dbs = _out.stdout.decode('utf-8').strip().splitlines()
|
||||
return()
|
||||
|
||||
def dump(self):
|
||||
if self.splitdumps:
|
||||
for db in self.dbs:
|
||||
args = copy.deepcopy(self.dumpopts)
|
||||
outfile = os.path.join(self.outdir, '{0}.sql'.format(db))
|
||||
if db in ('information_schema', 'performance_schema'):
|
||||
args.append('--skip-lock-tables')
|
||||
elif db == 'mysql':
|
||||
args.append('--flush-privileges')
|
||||
cmd = [self.mysqldumpbin,
|
||||
'--result-file={0}'.format(outfile)]
|
||||
cmd.extend(args)
|
||||
cmd.append(db)
|
||||
out = subprocess.run(cmd,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.PIPE)
|
||||
if out.returncode != 0:
|
||||
warn = ('Error dumping {0}: {1}').format(db, out.stderr.decode('utf-8').strip())
|
||||
warnings.warn(warn)
|
||||
else:
|
||||
outfile = os.path.join(self.outdir, 'all.databases.sql')
|
||||
args = copy.deepcopy(self.dumpopts)
|
||||
args.append('--result-file={0}'.format(outfile))
|
||||
if 'information_schema' in self.dbs:
|
||||
args.append('--skip-lock-tables')
|
||||
if 'mysql' in self.dbs:
|
||||
args.append('--flush-privileges')
|
||||
args.append(['--databases'])
|
||||
cmd = [self.mysqldumpbin]
|
||||
cmd.extend(args)
|
||||
cmd.extend(self.dbs)
|
||||
out = subprocess.run(cmd,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.PIPE)
|
||||
if out.returncode != 0:
|
||||
warn = ('Error dumping {0}: {1}').format(','.join(self.dbs),
|
||||
out.stderr.decode('utf-8').strip())
|
||||
warnings.warn(warn)
|
||||
return()
|
||||
229
plugins/yum_pkgs.py
Normal file
229
plugins/yum_pkgs.py
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
import datetime
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
##
|
||||
from lxml import etree
|
||||
try:
|
||||
# Note that currently, even on CentOS/RHEL 7, the yum module is only available for Python 2...
|
||||
# because reasons or something?
|
||||
# This may be re-done to allow for a third-party library in the case of python 3 invocation.
|
||||
import yum
|
||||
has_yum = True
|
||||
except ImportError:
|
||||
# This will get *ugly*. You have been warned. It also uses more system resources and it's INCREDIBLY slow.
|
||||
# But it's safe.
|
||||
# Requires yum-utils to be installed.
|
||||
# It assumes a python 3 environment for the exact above reason.
|
||||
import subprocess
|
||||
has_yum = False
|
||||
|
||||
# See <optools>:/storage/backups/borg/tools/restore_yum_pkgs.py to use the XML file this generates.
|
||||
|
||||
|
||||
# Detect RH version.
|
||||
ver_re =re.compile('^(centos.*|red\s?hat.*) ([0-9\.]+) .*$', re.IGNORECASE)
|
||||
# distro module isn't stdlib, and platform.linux_distribution() (AND platform.distro()) are both deprecated in 3.7.
|
||||
# So we get hacky.
|
||||
with open('/etc/redhat-release', 'r') as f:
|
||||
rawver = f.read()
|
||||
distver = [int(i) for i in ver_re.sub('\g<2>', rawver.strip()).split('.')]
|
||||
distname = re.sub('(Linux )?release', '', ver_re.sub('\g<1>', rawver.strip()), re.IGNORECASE).strip()
|
||||
# Regex pattern to get the repo name. We compile it just to speed up the execution.
|
||||
repo_re = re.compile('^@')
|
||||
# Python version
|
||||
pyver = sys.hexversion
|
||||
py3 = 0x30000f0 # TODO: check the version incompats
|
||||
|
||||
|
||||
class Backup(object):
|
||||
def __init__(self, explicit_only = True,
|
||||
include_deps = False,
|
||||
output = '~/.cache/backup/misc/installed_pkgs.xml'):
|
||||
self.explicit_only = explicit_only
|
||||
self.include_deps = include_deps
|
||||
self.reasons = []
|
||||
if self.explicit_only:
|
||||
self.reasons.append('user')
|
||||
if self.include_deps:
|
||||
self.reasons.append('dep')
|
||||
self.output = os.path.abspath(os.path.expanduser(output))
|
||||
if has_yum:
|
||||
self.yb = yum.YumBase()
|
||||
# Make it run silently.
|
||||
self.yb.preconf.debuglevel = 0
|
||||
self.yb.preconf.errorlevel = 0
|
||||
self.pkg_meta = []
|
||||
# TODO: XSD?
|
||||
self.pkgs = etree.Element('packages')
|
||||
self.pkgs.attrib['distro'] = distname
|
||||
self.pkgs.attrib['version'] = '.'.join([str(i) for i in distver])
|
||||
self.pkglist = b''
|
||||
self.getPkgList()
|
||||
self.buildPkgInfo()
|
||||
self.write()
|
||||
|
||||
def getPkgList(self):
|
||||
if has_yum:
|
||||
if not self.explicit_only:
|
||||
self.pkg_meta = self.yb.rpmdb.returnPackages()
|
||||
else:
|
||||
for pkg in self.yb.rpmdb.returnPackages():
|
||||
reason = pkg.yumdb_info.get('reason')
|
||||
if reason and reason.lower() in self.reasons:
|
||||
self.pkg_meta.append(pkg)
|
||||
else:
|
||||
pass # We do this in buildPkgInfo().
|
||||
return()
|
||||
|
||||
def buildPkgInfo(self):
|
||||
if not has_yum:
|
||||
def repoQuery(nevra, fmtstr):
|
||||
cmd = ['/usr/bin/repoquery',
|
||||
'--installed',
|
||||
'--queryformat', fmtstr,
|
||||
nevra]
|
||||
cmd_out = subprocess.run(cmd, stdout = subprocess.PIPE).stdout.decode('utf-8')
|
||||
return(cmd_out)
|
||||
_reason = '*'
|
||||
if self.reasons:
|
||||
if 'dep' not in self.reasons:
|
||||
_reason = 'user'
|
||||
cmd = ['/usr/sbin/yumdb',
|
||||
'search',
|
||||
'reason',
|
||||
_reason]
|
||||
rawpkgs = subprocess.run(cmd, stdout = subprocess.PIPE).stdout.decode('utf-8')
|
||||
reason_re = re.compile('^(\s+reason\s+=\s+.*|\s*)$')
|
||||
pkgs = []
|
||||
for line in rawpkgs.splitlines():
|
||||
if not reason_re.search(line):
|
||||
pkgs.append(line.strip())
|
||||
for pkg_nevra in pkgs:
|
||||
reponame = repo_re.sub('', repoQuery(pkg_nevra, '%{ui_from_repo}')).strip()
|
||||
repo = self.pkgs.xpath('repo[@name="{0}"]'.format(reponame))
|
||||
if repo:
|
||||
repo = repo[0]
|
||||
else:
|
||||
# This is pretty error-prone. Fix/cleanup your systems.
|
||||
repo = etree.Element('repo')
|
||||
repo.attrib['name'] = reponame
|
||||
rawrepo = subprocess.run(['/usr/bin/yum',
|
||||
'-v',
|
||||
'repolist',
|
||||
reponame],
|
||||
stdout = subprocess.PIPE).stdout.decode('utf-8')
|
||||
urls = []
|
||||
mirror = re.search('^Repo-mirrors\s*:', rawrepo, re.M)
|
||||
repostatus = re.search('^Repo-status\s*:', rawrepo, re.M)
|
||||
repourl = re.search('^Repo-baseurl\s*:', rawrepo, re.M)
|
||||
repodesc = re.search('^Repo-name\s*:', rawrepo, re.M)
|
||||
if mirror:
|
||||
urls.append(mirror.group(0).split(':', 1)[1].strip())
|
||||
if repourl:
|
||||
urls.append(repourl.group(0).split(':', 1)[1].strip())
|
||||
repo.attrib['urls'] = '>'.join(urls) # https://stackoverflow.com/a/13500078
|
||||
if repostatus:
|
||||
repostatus = repostatus.group(0).split(':', 1)[1].strip().lower()
|
||||
repo.attrib['enabled'] = ('true' if repostatus == 'enabled' else 'false')
|
||||
else:
|
||||
repo.attrib['enabled'] = 'false'
|
||||
if repodesc:
|
||||
repo.attrib['desc'] = repodesc.group(0).split(':', 1)[1].strip()
|
||||
else:
|
||||
repo.attrib['desc'] = '(metadata missing)'
|
||||
self.pkgs.append(repo)
|
||||
pkgelem = etree.Element('package')
|
||||
pkginfo = {'NEVRA': pkg_nevra,
|
||||
'desc': repoQuery(pkg_nevra, '%{summary}').strip()}
|
||||
# These are all values with no whitespace so we can easily combine into one call and then split them.
|
||||
(pkginfo['name'],
|
||||
pkginfo['release'],
|
||||
pkginfo['arch'],
|
||||
pkginfo['version'],
|
||||
pkginfo['built'],
|
||||
pkginfo['installed'],
|
||||
pkginfo['sizerpm'],
|
||||
pkginfo['sizedisk']) = re.split('\t',
|
||||
repoQuery(pkg_nevra,
|
||||
('%{name}\t'
|
||||
'%{release}\t'
|
||||
'%{arch}\t'
|
||||
'%{ver}\t' # version
|
||||
'%{buildtime}\t' # built
|
||||
'%{installtime}\t' # installed
|
||||
'%{packagesize}\t' # sizerpm
|
||||
'%{installedsize}') # sizedisk
|
||||
))
|
||||
for k in ('built', 'installed', 'sizerpm', 'sizedisk'):
|
||||
pkginfo[k] = int(pkginfo[k])
|
||||
for k in ('built', 'installed'):
|
||||
pkginfo[k] = datetime.datetime.fromtimestamp(pkginfo[k])
|
||||
for k, v in pkginfo.items():
|
||||
if pyver >= py3:
|
||||
pkgelem.attrib[k] = str(v)
|
||||
else:
|
||||
if isinstance(v, (int, long, datetime.datetime)):
|
||||
pkgelem.attrib[k] = str(v).encode('utf-8')
|
||||
elif isinstance(v, str):
|
||||
pkgelem.attrib[k] = v.decode('utf-8')
|
||||
else:
|
||||
pkgelem.attrib[k] = v.encode('utf-8')
|
||||
repo.append(pkgelem)
|
||||
else:
|
||||
for pkg in self.pkg_meta:
|
||||
reponame = repo_re.sub('', pkg.ui_from_repo)
|
||||
repo = self.pkgs.xpath('repo[@name="{0}"]'.format(reponame))
|
||||
if repo:
|
||||
repo = repo[0]
|
||||
else:
|
||||
repo = etree.Element('repo')
|
||||
repo.attrib['name'] = reponame
|
||||
try:
|
||||
repoinfo = self.yb.repos.repos[reponame]
|
||||
repo.attrib['urls'] = '>'.join(repoinfo.urls) # https://stackoverflow.com/a/13500078
|
||||
repo.attrib['enabled'] = ('true' if repoinfo in self.yb.repos.listEnabled() else 'false')
|
||||
repo.attrib['desc'] = repoinfo.name
|
||||
except KeyError: # Repo is missing
|
||||
repo.attrib['desc'] = '(metadata missing)'
|
||||
self.pkgs.append(repo)
|
||||
pkgelem = etree.Element('package')
|
||||
pkginfo = {'name': pkg.name,
|
||||
'desc': pkg.summary,
|
||||
'version': pkg.ver,
|
||||
'release': pkg.release,
|
||||
'arch': pkg.arch,
|
||||
'built': datetime.datetime.fromtimestamp(pkg.buildtime),
|
||||
'installed': datetime.datetime.fromtimestamp(pkg.installtime),
|
||||
'sizerpm': pkg.packagesize,
|
||||
'sizedisk': pkg.installedsize,
|
||||
'NEVRA': pkg.nevra}
|
||||
for k, v in pkginfo.items():
|
||||
if pyver >= py3:
|
||||
pkgelem.attrib[k] = str(v)
|
||||
else:
|
||||
if isinstance(v, (int, long, datetime.datetime)):
|
||||
pkgelem.attrib[k] = str(v).encode('utf-8')
|
||||
elif isinstance(v, str):
|
||||
pkgelem.attrib[k] = v.decode('utf-8')
|
||||
else:
|
||||
pkgelem.attrib[k] = v.encode('utf-8')
|
||||
repo.append(pkgelem)
|
||||
self.pkglist = etree.tostring(self.pkgs,
|
||||
pretty_print = True,
|
||||
xml_declaration = True,
|
||||
encoding = 'UTF-8')
|
||||
return()
|
||||
|
||||
def write(self):
|
||||
outdir = os.path.dirname(self.output)
|
||||
if pyver >= py3:
|
||||
os.makedirs(outdir, exist_ok = True)
|
||||
os.chmod(outdir, mode = 0o0700)
|
||||
else:
|
||||
if not os.path.isdir(outdir):
|
||||
os.makedirs(outdir)
|
||||
os.chmod(outdir, 0o0700)
|
||||
with open(self.output, 'wb') as f:
|
||||
f.write(self.pkglist)
|
||||
return()
|
||||
Loading…
Add table
Add a link
Reference in a new issue