#!/usr/bin/env python __version__ = 'Revision: GIT.139' import inspect import os import shlex, socket, stat, string, subprocess, sys, tarfile, time, zipfile import urllib.request try: import http.client as HTTPLIB except ModuleNotFoundError: import httplib as HTTPLIB try: from urllib.request import urlopen from urllib.parse import urlparse except ImportError: from urlparse import urlparse from urllib import urlopen # Globals wantedpip = '<=21,>7.1.2' driveurl = 'http://drive.astro-wise.org' def find_in_path(file_name): path = os.environ['PATH'] for d in path.split(os.pathsep): file_path = os.path.abspath(os.path.join(d, file_name)) if os.path.exists(file_path): return file_path return None def copyfile(src, dst, bufsiz=16384): fdsrc = open(src, 'rb') fddst = open(dst, 'wb') data = fdsrc.read(bufsiz) while data: fddst.write(data) data = fdsrc.read(bufsiz) fdsrc.close() fddst.close() os.chmod(dst, os.stat(src).st_mode) def lineno(): # Returns the current line number in our program. return inspect.currentframe().f_back.f_lineno def untar(file, path='.'): print('Unpacking %(file)s' % vars()) t = time.time() tar = tarfile.open(file,'r') dirname = tar.next().name if hasattr(tar, 'extractall'): tar.extractall(path=path) else: for member in tar.getmembers(): tar.extract(member, path=path) tar.close() t = time.time() - t print('Unpacking %(file)s took %(t)4.2f seconds' % vars()) return dirname.lstrip('/').split('/')[0] def docommand2(command): cd = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout result = cd.readlines() cd.close() if type(b'') != type(""): for j in range(len(result)): result[j] = str(result[j], "utf-8") return result def docommand(command, inlines=None): if inlines: sp = subprocess.Popen(shlex.split(command), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if type(inlines) == type(''): inlines = inlines.encode("latin-1") sp.stdin.write(inlines) sp.stdin.close() else: sp =subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) result = sp.stdout.readlines() sp.stdout.close() if type(b'') != type(""): for j in range(len(result)): result[j] = str(result[j], "latin-1") return result def dosystem(command): p = subprocess.Popen(command, shell=True) return os.waitpid(p.pid, 0)[1] def dodownload(url, filename=None, silent=False, check=False): assert url if not filename: filename = os.path.basename(url) if check and os.path.exists(filename): return if not silent: print('Downloading %(url)s --> %(filename)s' % vars()) t0 = time.time() urllib.request.urlretrieve(url, filename) if not silent: print('Download of %d bytes took %4.2f seconds' % (os.stat(filename).st_size, time.time() - t0)) def doupload(uploadurl, filename=None, silent=False): pieces = urlparse(uploadurl) if not filename: filename = pieces[2] if not silent: print('Uploading %(filename)s --> %(uploadurl)s' % vars()) t0 = time.time() c = 0 result = 500 try: conn = HTTPLIB.HTTPConnection(pieces[1]) size = os.stat(filename)[6] conn.putrequest('PUT', uploadurl) conn.putheader('Content-type', 'application/x-compressed') conn.putheader('Content-length', size) conn.endheaders() bs = 16384 fd = open(filename, 'rb') data = fd.read(bs) while data: conn.send(data) c += len(data) data = fd.read(bs) fd.close() r = conn.getresponse() result = r.status data = r.read() conn.close() except Exception as e: print('doupload exception: %s' % e) try: r = conn.getresponse() result = r.status data = r.read() conn.close() except: result = 1000 if not silent: print('Upload of %d bytes took %4.2f seconds' % (c, time.time() - t0)) return result >= 500 def checkawelinks(awehome, awearch, aweversion, awetarget, pythonver): aweexedir = os.path.join(awehome, awearch, aweversion, awetarget, 'bin') if not os.path.exists(aweexedir): os.makedirs(aweexedir) oldwd = os.getcwd() os.chdir(aweexedir) pythonexe = os.path.join(awehome, awearch, 'local', 'bin', 'python' + pythonver) # we need pythonexe present need = 'python' + pythonver.split('.')[0] if os.path.exists(need): if os.path.islink(need): os.remove(need) elif os.path.islink(need): os.remove(need) if not os.path.exists(need): copyfile(pythonexe, need) # make links for the rest for (s, d) in [(need, 'python'), (need, 'python' + pythonver.split('.')[0]), (need, 'python' + pythonver), (need, 'awe' + pythonver), ('python', 'awe')]: if d == s: continue if os.path.exists(d): if os.path.islink(d): if os.readlink(d) != d: os.remove(d) else: os.remove(d) if not os.path.exists(d): os.symlink(s, d) os.chdir(oldwd) def dositecustomize(awehome, awearch, sitecustomize, pythonver, awetarget='common', aweversion='master', force=False): if os.path.exists(sitecustomize): cwd = os.getcwd() os.chdir(os.path.join(awehome, awearch, 'local', 'lib', 'python' + pythonver)) fi = open(sitecustomize, 'r') line = fi.readline() firstline = '#DUMMY LINE\n' if os.path.exists('sitecustomize.py'): fd = open('sitecustomize.py', 'r') firstline = fd.readline() fd.close() if force or firstline != line: fo = open('sitecustomize.py', 'w') while line: line = line.replace("''", "'%s'" % awehome) line = line.replace("''", "'%s'" % awetarget) line = line.replace("''", "'%s'" % aweversion) fo.write(line) line = fi.readline() fo.close() fi.close() os.chdir(cwd) def whichos(*kw_list, **kw_dict): if 'AWEHOME' in os.environ: awehome = os.environ['AWEHOME'] elif 'awehome' in kw_dict: awehome = kw_dict['awehome'] else: awehome = os.path.join(os.getcwd(), 'awehome') awehome.rstrip(os.path.sep) if 'AWEVERSION' in os.environ: aweversion = os.environ['AWEVERSION'] elif 'aweversion' in kw_dict: aweversion = kw_dict['aweversion'].split(',')[0] else: aweversion = 'develop' argdefaults = { 'awehome': awehome, 'version': aweversion, } argcount = 0 for arg in argdefaults.keys(): if arg in kw_dict.keys(): val = kw_dict[arg] argcount += 1 else: val = argdefaults[arg] if arg == 'awehome': awehome = val awehome.rstrip(os.path.sep) elif arg == 'version': aweversion = val else: raise Exception('Something wrong with %(arg)s [%(val)s]' % vars()) if kw_list or argcount != len(kw_dict) : keyorder = ['awehome', 'version'] print('''\ Name: whichos Purpose: Determines the Astro-Wise Operating System name. Usage: %s %s= %s= root of install AWEHOME [%s] version [%s] Version: %s ''' % tuple([sys.argv[0]] + keyorder + [argdefaults[k] for k in keyorder] + __version__.split()[1:2])) sys.exit(0) try: from common.util.targetplatform import targetplatform r = targetplatform() except: if os.path.exists('targetplatform.py'): os.remove('targetplatform.py') cleanup = False if 'awehome' in kw_dict and os.path.exists(os.path.join(kw_dict['awehome'], aweversion, 'common', 'util', 'targetplatform.py')): sys.path.insert(0, os.path.join(kw_dict['awehome'], aweversion, 'common', 'util')) else: dodownload(driveurl+'/awesoft/targetplatform.py', 'targetplatform.py') cleanup = True sys.path.insert(0, os.getcwd()) from targetplatform import targetplatform r = targetplatform() del(sys.path[0]) if cleanup: if os.path.exists('targetplatform.py'): os.remove('targetplatform.py') if os.path.exists('targetplatform.pyc'): os.remove('targetplatform.pyc') if os.path.exists('targetplatform.pyo'): os.remove('targetplatform.pyo') return r def maketxttarball(awehome, awetarget, versions, uploadurl=''): ''' maketxttarball makes a tarball of the python code and uploads this to the uploadurl ''' def tarfilter(tarinfo): if tarinfo.name.endswith('.py0'): return None if tarinfo.name.endswith('.pyc'): return None return tarinfo for v in versions: tbname = 'awetxt-%(awetarget)s-%(v)s.tgz' % vars() dirs = [os.path.join(v, 'common')] if awetarget != 'common': dirs.append(os.path.join(v, awetarget)) print('Creating %s' % tbname) t0 = time.time() tar = tar.open(tbname, 'w:gz') for dir in dirs: tar.add(os.path.join(awehome, dir), arcname=dir, filter=tarfilter) tar.close() print('Create of %s took %4.2f seconds' % (tbname, time.time() - t0)) if uploadurl: uplname = 'awetxt-%(awetarget)s-%(v)s.tgz' % vars() try: r = doupload('%(uploadurl)s/%(uplname)s' % vars(), tbname) except Exception as e: print('Trying curl: %s' % e) uplcmd = 'curl -f -T %(tbname)s %(uploadurl)s/%(uplname)s' % vars() r = dosystem(uplcmd) if r: prevtbname = tbname tname = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())) uplname = 'awetxt-%(awetarget)s-%(tname)s-%(v)s.tgz' % vars() print('Probably the file %(prevtbname)s already existed, trying %(uplname)s' % vars()) try: r = doupload('%(uploadurl)s/%(uplname)s' % vars(), tbname) except Exception as e: print('Trying curl: %s' % e) uplcmd = 'curl -f -T %(tbname)s %(uploadurl)s/%(uplname)s' % vars() r = dosystem(uplcmd) def makebintarball(awehome='', awearch=None, awetarget='common', versions=['master', 'develop'], uploadurl=''): if not awearch: awearch = whichos(awehome=awehome) discoveredpythonversions = [i[3:] for i in os.listdir(os.path.join(awehome, awearch, 'local', 'bin')) if i.startswith('awe') and len(i) > 3] if not discoveredpythonversions: print('No awe executables found!') sys.exit(1) discoveredpythonversions.sort() for pyver in discoveredpythonversions: exactpythonversion = docommand('env -uPYTHONVERBOSE ' + os.path.join(awehome, awearch, 'local', 'bin', 'awe' + pyver) + ' -V')[-1].split()[-1] ipv = [int(n) for n in exactpythonversion.split('.')] usepip = ipv[0] > 2 or ipv[2] > 10 for v in versions: if ipv[0] > 2: tbname = 'awebin-%(awetarget)s-%(v)s-%(pyver)s-%(awearch)s.tgz' % vars() else: tbname = 'awebin-%(awetarget)s-%(v)s-%(awearch)s.tgz' % vars() dirs = [] if usepip: if os.path.exists(os.path.join(awehome, awearch, v, awetarget)): dirs.append(os.path.join(awearch, v, awetarget)) else: dirs.append(os.path.join(awearch, v, 'common')) if awetarget != 'common' and os.path.exists(os.path.join(awehome, awearch, v, awetarget)): dirs.append(os.path.join(awearch, v, awetarget)) for d in os.listdir(os.path.join(awehome, awearch)): if d == 'local': dirs.append(os.path.join(awearch, d)) elif d.startswith('instantclient'): dirs.append(os.path.join(awearch, d)) print('Creating %s' % tbname) t0 = time.time() tar = tarfile.open(tbname, 'w:gz') for dir in dirs: tar.add(os.path.join(awehome, dir), arcname=dir) tar.close() print('Create of %s took %4.2f seconds' % (tbname, time.time() - t0)) if uploadurl: if ipv[0] > 2: uplname = 'awebin-%(awetarget)s-test-%(v)s-%(pyver)s-%(awearch)s.tgz' % vars() else: uplname = 'awebin-%(awetarget)s-test-%(v)s-%(awearch)s.tgz' % vars() try: r = doupload('%(uploadurl)s/%(uplname)s' % vars(), tbname) except Exception as e: print('Trying curl: %s' % e) uplcmd = 'curl -f -T %(tbname)s %(uploadurl)s/%(uplname)s' % vars() r = dosystem(uplcmd) if r: prevtbname = tbname tname = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time())) uplname = 'awebin-%(awetarget)s-test-%(tname)s-%(v)s-%(awearch)s.tgz' % vars() print('Probably the file %(prevtbname)s already existed, trying %(uplname)s' % vars()) try: r = doupload('%(uploadurl)s/%(uplname)s' % vars(), tbname) except Exception as e: print('Trying curl: %s' % e) uplcmd = 'curl -f -T %(tbname)s %(uploadurl)s/%(uplname)s' % vars() r = dosystem(uplcmd) def restartawesetup(newpythonexe, startdir, thisthing): if os.path.exists(newpythonexe) and os.path.dirname(os.path.realpath(sys.executable)) != os.path.dirname(os.path.realpath(newpythonexe)): print('Restarting "%(thisthing)s" with correct executable "%(newpythonexe)s" in directory "%(startdir)s"!' % vars()) os.chdir(startdir) os.execl(newpythonexe, newpythonexe, thisthing, *sys.argv[1:]) def awesetup(*kw_list, **kw_dict): awearchoverrule = False startdir = os.getcwd() bzip2url = 'http://drive.astro-wise.org/awesoft/bzip2/bzip2-1.0.6.tar.gz' opensslurl = 'http://drive.astro-wise.org/awesoft/openssl/openssl-1.1.1m.tar.gz' readlineurl = 'http://ftp.gnu.org/pub/gnu/readline/readline-6.2.tar.gz' tclurl = 'http://drive.astro-wise.org/awesoft/tcl/tcl8.5.7-src.tar.gz' tkurl = 'http://drive.astro-wise.org/awesoft/tk/tk8.5.7-src.tar.gz' zliburl = 'http://drive.astro-wise.org/awesoft/zlib/zlib-1.2.5.tar.gz' thisthing = os.path.realpath(sys.argv[0]) wantedpipversion = wantedpip if 'AWEHOME' in os.environ: awehome = os.environ['AWEHOME'] else: awehome = os.path.join(os.getcwd(), 'awehome') if 'AWETARGET' in os.environ: awetarget = os.environ['AWETARGET'] else: awetarget = 'common' awehome.rstrip(os.path.sep) if 'awearch' in kw_dict: awearch = kw_dict['awearch'] awearchoverrule = True else: if 'awehome' in kw_dict: awearch = whichos(awehome=kw_dict['awehome']) else: awearch = whichos(awehome=awehome) gitprivatekeytargets = ['install', 'common', 'astro', 'euclid', 'easdss', 'dataserver'] defaultrs = 'git' # repo system = git defaultgitpath = find_in_path('git') git_passfile = '' git_path = '' defaultcvsroot = ':pserver:anoncvs@cvs.astro-wise.org:/cvsroot' defaultgitlab = 'git@gitlab.astro-wise.org' versions = ['master', 'develop'] # Use for default current awe version if awe is running awesetup for updates if os.path.basename(sys.executable).startswith('awe'): defaultpythonversion = sys.version.split()[0] else: defaultpythonversion = '3.9' argdefaults = { 'rs' : defaultrs, 'awehome' : awehome, 'awearch' : awearch, 'awetarget' : awetarget, 'pythonversion' : defaultpythonversion, 'oracleversion' : '11.2', 'cvsroot' : defaultcvsroot, 'targetcvsroot' : '', 'versions' : ','.join(versions), 'branches' : ','.join(versions), 'gitlab' : defaultgitlab, 'downloadurl' : driveurl+'/awesoft/grid-install', 'skip' : 'nothing', 'gitrsaprvkey' : git_passfile, 'gitpath' : defaultgitpath, } argcount = 0 usepip = 0 for arg in argdefaults.keys(): usedefault = True if arg in kw_dict.keys(): argcount += 1 val = kw_dict[arg] usedefault = False else: val = argdefaults[arg] if arg == 'rs': rs = val if rs not in ['cvs', 'git']: raise Exception('Unknown Repository System "%(rs)s"' % vars()) if arg == 'awehome': awehome = os.path.abspath(val) if awehome == os.path.abspath(os.getcwd()): raise Exception('Not allowed to install in current directory!') awehome.rstrip(os.path.sep) elif arg == 'awetarget': if '/' in val: awetarget = val else: if val == 'euclid': awetarget = 'euclid/' + val else: awetarget = 'omegacen/' + val elif arg == 'awearch': awearch = val elif arg == 'pythonversion': ipv = [int(n) for n in val.split('.')] if val.startswith('2.3'): pythonver = '2.3' if val == pythonver: pythonurl = driveurl+'/awesoft/Python-2.3.6.tgz' else: pythonurl = driveurl+'/awesoft/python/Python-'+val+'.tgz' pythonpatchurls = [ driveurl+'/awesoft/Python-2.3.6-readline.patch', driveurl+'/awesoft/Python-2.3.6-setup.patch' ] elif val.startswith('2.5'): pythonver = '2.5' if val == pythonver: val = '2.5.5' pythonurl = driveurl+'/awesoft/python/Python-'+val+'.tgz' pythonpatchurls = [] elif val.startswith('2.7'): pythonver = '2.7' if val == pythonver: val = '2.7.11' pythonurl = driveurl+'/awesoft/python/Python-'+val+'.tgz' pythonpatchurls = [] if int(val[4:]) > 10: usepip = 1 elif val.startswith('3.0') or (val.startswith('3.1') and not val.startswith('3.10')) or val.startswith('3.2') or val.startswith('3.3') or val.startswith('3.4'): raise Exception('No support for python "%s"!' % (val)) elif ipv[0] == 3 and ipv[1] > 4: pythonver = '%d.%d' % (ipv[0], ipv[1]) if val == '3.5': val = '3.5.10' if val == '3.6': val = '3.6.15' if val == '3.7': val = '3.7.13' if val == '3.8': val = '3.8.13' if val == '3.9': val = '3.9.13' if val == '3.10': val = '3.10.5' pythonurl = driveurl+'/awesoft/python/Python-'+val+'.tgz' pythonpatchurls = [] usepip = 2 else: pythonver = '.'.join(val.split('.')[:2]) pythonurl = driveurl+'/awesoft/python/Python-'+val+'.tgz' pythonpatchurls = [] pythonversion = val ipv = [int(n) for n in val.split('.')] elif arg == 'oracleversion': if awearch.split('-')[0]+'.'+awearch.split('-')[-1] in ['Linux.i386', 'Linux.i686']: if val == '11.2': instantclienturls = [ driveurl+'/instantclient/instantclient-basic-linux-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sdk-linux-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sqlplus-linux-11.2.0.3.0.zip' ] instantclientdir = 'instantclient_11_2' else: instantclienturls = [ driveurl+'/instantclient/instantclient-basic-linux32-11.1.0.7.zip', driveurl+'/instantclient/instantclient-sdk-linux32-11.1.0.7.zip', driveurl+'/instantclient/instantclient-sqlplus-linux32-11.1.0.7.zip' ] instantclientdir = 'instantclient_11_1' elif awearch.split('-')[0]+'.'+awearch.split('-')[-1] == 'Linux.x86_64': if val == '11.2': instantclienturls = [ driveurl+'/instantclient/instantclient-basic-linux.x64-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sdk-linux.x64-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sqlplus-linux.x64-11.2.0.3.0.zip' ] instantclientdir = 'instantclient_11_2' else: instantclienturls = [ driveurl+'/instantclient/basic-11.1.0.70-linux-x86_64.zip', driveurl+'/instantclient/sdk-11.1.0.7.0-linux-x86_64.zip', driveurl+'/instantclient/sqlplus-11.1.0.7.0-linux-x86_64.zip' ] instantclientdir = 'instantclient_11_1' elif awearch.split('-')[0]+'.'+awearch.split('-')[-1] in ['Darwin.i386', 'Darwin.i686']: if val == '11.2': instantclienturls = [ driveurl+'/instantclient/instantclient-basic-macos.x32-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sdk-macos.x32-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sqlplus-macos.x32-11.2.0.3.0.zip', ] instantclientdir = 'instantclient_11_2' else: instantclienturls = [ driveurl+'/instantclient/instantclient-basic-10.2.0.4.0-macosx-x86.zip', driveurl+'/instantclient/instantclient-sdk-10.2.0.4.0-macosx-x86.zip', driveurl+'/instantclient/instantclient-sqlplus-10.2.0.4.0-macosx-x86.zip', ] instantclientdir = 'instantclient' elif awearch.split('-')[0]+'.'+awearch.split('-')[-1] == 'Darwin.x86_64': if val == '11.2': instantclienturls = [ driveurl+'/instantclient/instantclient-basic-macos.x64-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sdk-macos.x64-11.2.0.3.0.zip', driveurl+'/instantclient/instantclient-sqlplus-macos.x64-11.2.0.3.0.zip', ] instantclientdir = 'instantclient_11_2' else: instantclienturls = [ driveurl+'/instantclient/instantclient-basic-10.2.0.4.0-macosx-x64.zip', driveurl+'/instantclient/instantclient-sdk-10.2.0.4.0-macosx-x64.zip', driveurl+'/instantclient/instantclient-sqlplus-10.2.0.4.0-macosx-x64.zip' ] instantclientdir = 'instantclient' else: raise Exception('No Oracle client software available for this targetplatform: %(awearch)s!' % vars()) elif arg == 'cvsroot': cvsroot = val elif arg == 'targetcvsroot': targetcvsroot = val elif arg in ['branches', 'versions']: if not usedefault: versions = [] for v in val.split(','): if v not in versions: versions.append(v) elif arg == 'gitlab': gitlab = val elif arg == 'downloadurl': downloadurl = val.rstrip('/') elif arg == 'skip': skiplist = val.split(',') elif arg == 'gitrsaprvkey': git_passfile = val elif arg == 'gitpath': git_path = val if kw_list or len(kw_dict) != argcount: keyorder = ['awehome', 'awetarget', 'pythonversion', 'oracleversion', 'rs', 'cvsroot', 'targetcvsroot', 'versions', 'gitlab', 'branches', 'downloadurl', 'awearch', 'skip', 'gitrsaprvkey', 'gitpath'] assert all(i in keyorder for i in argdefaults.keys()) print('''\ Name: awesetup Purpose: Install astro-wise software! Usage: %s %s= %s= %s= %s= %s= %s= %s= %s= %s= %s=
%s= %s= %s= %s= %s= root of install AWEHOME [%s] wanted AWETARGET distribution i.e. common, astro, awlofar [%s] awetarget maybe preceded by a gitgroupname followed by a /, default group name = omegacen python version [%s] oracle client version [%s] repo system, cvs or git [%s] cvsroot [%s] cvsroot for AWETARGET distribution [%s] cvs versions [%s] gitlab [%s]
gitlab branches [%s] downloadurl [%s] AWEARCH (for overruling default, could be DANGEROUS!)[%s] comma separated skip list, can be all, base and [%s] skip=all means just install the python sources, skip=base means do not compile python executable location of ssh rsa private key to use for git authentication [%s] path to git (version >= 2) [%s] Note: starting from April 1, 2017 the defaults for repo system will be changed from cvs to git Version: %s ''' % tuple([sys.argv[0]] + keyorder + [argdefaults[k] for k in keyorder] + __version__.split()[1:2])) sys.exit(0) if rs == 'git': if not os.path.exists(git_path): raise Exception('git not found "%(git_path)s"' % vars()) pythondir = os.path.basename(pythonurl)[:-4] def touch(file): fd = open(file, 'a') fd.close() def unzip(file, dir=None): print('Unpacking %(file)s' % vars()) t = time.time() zip = zipfile.ZipFile(file, mode="r") for file in zip.namelist(): dirpart, namepart = os.path.split(file) if dir: dir1 = dirpart l = 0 while dir1: l = len(dir1) dir1 = os.path.dirname(dir1) if os.path.isabs(dirpart[l:]): dirpart = dirpart[l+1:] else: dirpart = dirpart[l:] dirpart = os.path.join(dir, dirpart) if dirpart and not os.path.exists(dirpart): os.makedirs(dirpart) if namepart: fd = open(os.path.join(dirpart, namepart), "wb") fd.write(zip.read(file)) fd.close() zip.close() t = time.time() - t print('Unpacking %(file)s took %(t)4.2f seconds' % vars()) def getdir(file): tar = tarfile.open(file,'r') dirname = tar.next().name tar.close() return dirname.lstrip('/').split('/')[0] def checkcvspass(cvs_passfile): text = '/1 :pserver:anoncvs@cvs.astro-wise.org:2401/cvsroot AyZ, 03?>d\n' if not os.path.exists(cvs_passfile): fd = os.open(cvs_passfile, os.O_CREAT | os.O_WRONLY, stat.S_IREAD| stat.S_IWRITE) os.write(fd, text.encode()) os.close(fd) else: fd = open(cvs_passfile, 'r+') append = True for line in fd.readlines(): if line == text: append = False if append: fd.write(text) fd.close() def checkgitpass(git_passfile): data = \ b'\x2d\x2d\x2d\x2d\x2d\x42\x45\x47\x49\x4e\x20\x52\x53\x41\x20\x50\x52\x49\x56\x41\x54\x45\x20\x4b\x45\x59\x2d\x2d\x2d\x2d\x2d\x0a\x4d\x49\x49\x45\x6f\x51\x49\x42\x41\x41\x4b\x43\x41\x51\x45\x41\x79\x51' +\ b'\x76\x70\x4b\x59\x52\x38\x2b\x54\x43\x64\x4d\x4e\x4b\x70\x58\x54\x53\x69\x5a\x6b\x48\x62\x79\x71\x30\x44\x69\x5a\x4b\x42\x61\x71\x41\x58\x4d\x33\x65\x30\x44\x35\x44\x49\x2b\x41\x73\x4d\x0a\x6e\x2b\x64' +\ b'\x31\x75\x48\x78\x49\x54\x50\x6d\x39\x66\x35\x43\x30\x77\x37\x56\x77\x67\x6e\x7a\x50\x42\x79\x5a\x37\x43\x31\x4c\x71\x36\x35\x52\x48\x69\x44\x2f\x46\x56\x55\x71\x5a\x69\x50\x47\x32\x38\x30\x63\x43\x39' +\ b'\x56\x37\x75\x4d\x62\x51\x43\x76\x79\x6b\x71\x0a\x36\x37\x31\x4a\x75\x70\x68\x30\x4f\x31\x39\x43\x43\x56\x4c\x59\x4a\x77\x76\x48\x45\x52\x72\x32\x2f\x65\x79\x75\x49\x67\x64\x71\x57\x77\x47\x75\x47\x66' +\ b'\x61\x44\x4f\x74\x36\x57\x45\x79\x6f\x51\x73\x61\x51\x32\x6b\x6e\x6a\x64\x52\x43\x30\x57\x32\x36\x76\x46\x0a\x42\x65\x6f\x4a\x32\x72\x6b\x4a\x79\x75\x35\x65\x65\x50\x53\x68\x45\x66\x44\x6b\x43\x43\x74' +\ b'\x69\x50\x37\x35\x74\x7a\x2b\x46\x48\x7a\x45\x62\x63\x61\x43\x42\x46\x66\x50\x71\x6d\x48\x54\x37\x6c\x45\x36\x4e\x67\x49\x55\x6b\x59\x41\x53\x6e\x32\x77\x31\x63\x75\x0a\x5a\x6e\x67\x4b\x55\x37\x79\x58' +\ b'\x68\x32\x61\x74\x49\x2b\x46\x53\x44\x62\x4b\x72\x33\x39\x2b\x2b\x4d\x68\x50\x74\x53\x2b\x55\x37\x6b\x51\x44\x64\x4b\x46\x49\x67\x62\x58\x57\x77\x71\x70\x78\x6f\x68\x36\x75\x76\x71\x68\x39\x78\x34\x30' +\ b'\x54\x2b\x7a\x56\x43\x38\x0a\x71\x72\x71\x43\x59\x61\x6b\x6f\x41\x6c\x6e\x4c\x63\x74\x63\x4c\x34\x2b\x56\x50\x7a\x47\x67\x50\x4d\x72\x53\x41\x52\x43\x6d\x71\x57\x63\x4c\x32\x58\x51\x49\x42\x49\x77\x4b' +\ b'\x43\x41\x51\x42\x68\x70\x72\x4d\x55\x4b\x6d\x69\x57\x53\x74\x41\x42\x78\x57\x67\x30\x0a\x6c\x65\x68\x36\x7a\x34\x67\x44\x57\x31\x6c\x39\x56\x63\x6e\x55\x74\x43\x69\x47\x74\x6e\x77\x48\x6a\x33\x65\x4f' +\ b'\x61\x38\x52\x4e\x71\x76\x41\x4a\x4a\x6d\x78\x43\x70\x54\x41\x6f\x42\x48\x55\x4f\x6d\x66\x54\x52\x71\x6c\x58\x74\x68\x37\x67\x62\x63\x57\x72\x4b\x0a\x4d\x68\x51\x64\x6d\x31\x45\x34\x44\x6b\x71\x53\x2b' +\ b'\x51\x2b\x33\x2f\x65\x74\x2b\x66\x70\x44\x7a\x6b\x66\x4b\x30\x6e\x76\x65\x58\x45\x73\x77\x4b\x4c\x4d\x71\x39\x77\x49\x33\x4b\x41\x36\x72\x52\x49\x76\x70\x4b\x49\x77\x70\x41\x30\x67\x74\x76\x6e\x54\x4f' +\ b'\x6f\x0a\x69\x38\x6d\x58\x6c\x50\x33\x70\x5a\x4d\x79\x4d\x39\x79\x79\x75\x44\x65\x36\x65\x36\x6a\x2b\x57\x49\x36\x72\x39\x4c\x67\x63\x68\x44\x43\x52\x43\x32\x76\x45\x77\x56\x73\x76\x36\x43\x71\x4f\x2f' +\ b'\x6a\x47\x37\x61\x32\x33\x4c\x4b\x48\x32\x6e\x43\x54\x59\x34\x31\x0a\x36\x52\x73\x64\x61\x30\x65\x53\x4c\x77\x64\x55\x57\x52\x30\x73\x61\x4c\x33\x36\x4f\x36\x4c\x32\x6a\x46\x6c\x31\x4f\x4e\x62\x52\x57' +\ b'\x57\x69\x51\x62\x44\x6a\x6a\x4e\x66\x57\x6f\x4f\x36\x4b\x77\x63\x65\x45\x6b\x45\x78\x51\x2f\x57\x63\x42\x2f\x64\x68\x54\x6f\x0a\x57\x6c\x48\x35\x4c\x6f\x56\x49\x4c\x6d\x6d\x6e\x51\x71\x41\x4a\x4d\x55' +\ b'\x34\x57\x47\x33\x71\x51\x33\x58\x4e\x36\x4b\x63\x51\x6f\x71\x32\x54\x36\x4e\x59\x59\x4c\x32\x6c\x51\x65\x35\x6a\x6a\x56\x6e\x57\x4e\x53\x67\x43\x63\x5a\x78\x5a\x46\x4c\x53\x63\x59\x75\x0a\x71\x6b\x76' +\ b'\x4c\x41\x6f\x47\x42\x41\x4f\x6a\x47\x45\x66\x4d\x74\x4d\x65\x75\x32\x36\x37\x69\x71\x66\x49\x48\x66\x69\x54\x5a\x64\x64\x42\x6f\x34\x61\x55\x77\x43\x68\x32\x70\x49\x65\x46\x4c\x31\x78\x46\x77\x69\x65' +\ b'\x32\x4d\x63\x41\x4b\x2b\x69\x76\x47\x6d\x31\x0a\x53\x78\x6b\x58\x30\x78\x44\x32\x4f\x4e\x4e\x39\x43\x4c\x57\x78\x55\x6b\x64\x50\x73\x75\x73\x6d\x42\x76\x58\x6f\x43\x61\x33\x4b\x2f\x4b\x65\x6a\x63\x5a' +\ b'\x65\x39\x6a\x43\x2f\x6e\x38\x59\x45\x65\x56\x39\x4a\x68\x35\x35\x41\x65\x59\x71\x66\x46\x7a\x6a\x69\x79\x0a\x68\x73\x55\x50\x5a\x56\x55\x2f\x6b\x4c\x61\x63\x42\x50\x46\x54\x4e\x74\x4c\x46\x4c\x4e\x5a' +\ b'\x39\x49\x5a\x4a\x65\x4f\x32\x57\x6a\x79\x63\x48\x76\x50\x66\x35\x65\x47\x67\x6a\x50\x69\x64\x68\x55\x32\x34\x49\x52\x41\x6f\x47\x42\x41\x4e\x30\x62\x61\x44\x75\x6a\x0a\x76\x54\x7a\x6c\x63\x2f\x63\x6d' +\ b'\x56\x63\x41\x74\x4e\x32\x54\x65\x31\x43\x48\x51\x37\x7a\x6a\x33\x62\x73\x39\x76\x32\x46\x65\x55\x73\x61\x4c\x4d\x31\x78\x4c\x4b\x6d\x53\x4f\x66\x72\x4f\x54\x7a\x32\x76\x77\x6b\x5a\x4b\x68\x34\x77\x52' +\ b'\x2f\x4e\x63\x41\x6d\x36\x0a\x31\x78\x6a\x70\x77\x58\x4e\x2b\x68\x7a\x58\x77\x69\x67\x6f\x45\x6e\x6d\x48\x6a\x47\x62\x46\x35\x6f\x63\x65\x6f\x30\x33\x46\x49\x2f\x69\x58\x46\x5a\x53\x4f\x4e\x53\x68\x46' +\ b'\x42\x4a\x75\x5a\x59\x7a\x6e\x63\x44\x7a\x4e\x55\x41\x2b\x32\x6d\x42\x45\x2b\x77\x77\x0a\x39\x61\x4e\x37\x4f\x30\x79\x5a\x2f\x6b\x43\x59\x77\x61\x6d\x31\x66\x53\x72\x4a\x35\x66\x79\x51\x66\x43\x34\x7a' +\ b'\x2b\x4e\x2b\x6b\x31\x43\x4f\x4e\x41\x6f\x47\x41\x54\x38\x37\x68\x6c\x54\x51\x52\x48\x5a\x33\x4e\x4b\x56\x38\x47\x48\x65\x59\x39\x72\x44\x31\x4d\x0a\x59\x4d\x4c\x69\x52\x66\x49\x39\x44\x6e\x38\x2f\x4d' +\ b'\x6d\x4c\x6b\x50\x4e\x69\x66\x56\x53\x34\x73\x48\x76\x58\x33\x64\x4c\x4d\x76\x73\x4e\x54\x33\x36\x49\x37\x75\x36\x57\x79\x79\x68\x32\x2f\x2b\x38\x2b\x44\x50\x6f\x52\x52\x61\x4b\x47\x7a\x51\x48\x6c\x51' +\ b'\x35\x0a\x58\x67\x77\x6d\x38\x6a\x4a\x62\x38\x79\x72\x7a\x74\x7a\x32\x61\x64\x41\x52\x50\x5a\x4a\x79\x30\x48\x45\x50\x52\x72\x51\x6f\x43\x55\x6a\x45\x71\x44\x70\x6c\x7a\x63\x63\x38\x58\x6f\x7a\x4a\x35' +\ b'\x4d\x6c\x49\x37\x51\x6a\x6d\x48\x32\x6d\x6c\x7a\x63\x30\x34\x5a\x0a\x53\x63\x35\x65\x5a\x64\x63\x65\x33\x6e\x4d\x4b\x73\x4a\x49\x66\x58\x38\x73\x43\x67\x59\x42\x34\x42\x35\x42\x61\x34\x39\x76\x42\x2b' +\ b'\x4f\x63\x75\x5a\x55\x52\x2b\x52\x47\x36\x48\x4e\x79\x4b\x7a\x52\x59\x6b\x74\x6a\x61\x4b\x4e\x33\x61\x46\x55\x48\x59\x55\x41\x0a\x6d\x78\x57\x72\x48\x59\x5a\x56\x4b\x73\x75\x53\x4f\x7a\x78\x63\x2f\x64' +\ b'\x41\x2b\x4d\x75\x78\x2b\x2b\x6e\x36\x6d\x4d\x6a\x70\x41\x75\x57\x6b\x45\x4c\x72\x35\x74\x76\x52\x65\x38\x53\x36\x5a\x76\x70\x79\x73\x30\x64\x54\x71\x51\x39\x54\x2b\x56\x52\x4f\x47\x38\x0a\x75\x35\x31' +\ b'\x4e\x30\x46\x74\x6f\x63\x39\x4e\x4a\x32\x48\x41\x55\x75\x4f\x75\x4a\x6b\x74\x47\x66\x72\x48\x69\x48\x69\x45\x72\x56\x46\x77\x4c\x76\x45\x63\x53\x4a\x64\x33\x38\x53\x2f\x42\x39\x67\x58\x76\x6b\x78\x56' +\ b'\x62\x2f\x42\x54\x32\x6e\x59\x66\x67\x7a\x43\x0a\x31\x77\x4b\x42\x67\x51\x43\x4c\x55\x46\x45\x5a\x46\x67\x79\x41\x77\x37\x74\x71\x50\x33\x76\x61\x6a\x67\x53\x65\x77\x76\x71\x6f\x71\x45\x73\x37\x63\x69' +\ b'\x4c\x49\x30\x48\x50\x68\x54\x65\x37\x41\x77\x6a\x2f\x75\x38\x6c\x4e\x2f\x68\x34\x58\x70\x75\x77\x2f\x53\x0a\x4c\x4a\x73\x6f\x6f\x63\x57\x50\x43\x5a\x54\x43\x44\x4c\x62\x76\x7a\x66\x4e\x6f\x37\x78\x2f' +\ b'\x35\x77\x67\x34\x4e\x6b\x65\x62\x6d\x4e\x55\x55\x44\x4f\x31\x61\x54\x2b\x31\x77\x36\x62\x6d\x6a\x52\x2f\x57\x36\x31\x39\x77\x2b\x54\x6e\x7a\x2b\x36\x68\x64\x34\x37\x0a\x57\x6b\x67\x54\x54\x48\x39\x54' +\ b'\x56\x77\x6c\x47\x49\x66\x6b\x52\x67\x50\x49\x5a\x65\x36\x52\x75\x6e\x6a\x59\x36\x37\x6f\x74\x4d\x42\x4a\x56\x6e\x49\x36\x34\x5a\x53\x48\x6b\x43\x43\x44\x67\x31\x68\x51\x3d\x3d\x0a\x2d\x2d\x2d\x2d\x2d' +\ b'\x45\x4e\x44\x20\x52\x53\x41\x20\x50\x52\x49\x56\x41\x54\x45\x20\x4b\x45\x59\x2d\x2d\x2d\x2d\x2d\x0a' if not os.path.exists(git_passfile): fd = os.open(git_passfile, os.O_CREAT | os.O_WRONLY, stat.S_IREAD| stat.S_IWRITE) os.write(fd, data) os.close(fd) def checkawesetupversion(awecodepath): awesetup = os.path.join(awecodepath, 'install', 'util', 'awesetup.py') if os.path.exists(awesetup): fd = open(awesetup, 'r') line = fd.readline() while line and not line.startswith('__version__'): line = fd.readline() fd.close() if line: line = line[:-1].split('=')[1].strip().strip("'") nv = int(line.split()[1].split('.')[-1]) ov = int(__version__.split()[1].split('.')[-1]) if nv > ov: print('''\ ********************************************************** There is a new version of awesetup available! Try running: %s %s ********************************************************** ''' % (awesetup, ' '.join(sys.argv[1:]))) sys.exit(0) def testpythonmodule(name): T = 'True' F = 'False' pycode = '''\ try: import %(name)s print('%(T)s') except: print('%(F)s') ''' % vars() pexe = 'python.exe' if not os.path.exists(pexe): pexe = 'python' r = docommand('env -uPYTHONVERBOSE ./%(pexe)s' % vars(), pycode)[0] return r.find(T) > -1 and r.find(F) == -1 def install_local(url, awelocal, config_opts='', commands=[]): savecwd = os.getcwd() os.chdir(awebuild) if not os.path.exists(os.path.basename(url)): dodownload(url) dir = untar(os.path.basename(url)) else: dir = getdir(os.path.basename(url)) os.chdir(dir) if commands: c = 0 for command in commands: c += 1 print('Execute: %(command)s' % vars()) dosystem('%(command)s > ../%(dir)s_%(c)d.log 2> ../%(dir)s_%(c)d.err' % vars()) os.chdir(savecwd) return if (os.path.exists('./configure') or os.path.exists('./unix/configure')) and not os.path.exists('.__configure_done'): cmd = './configure --prefix=%(awelocal)s %(config_opts)s' % vars() if not os.path.exists('./configure'): cmd = '(cd unix ; ' + cmd + ')' print('Configuring %(dir)s: %(cmd)s' % vars()) dosystem('%(cmd)s > ../%(dir)s_configure.log 2> ../%(dir)s_configure.err' % vars()) touch('.__configure_done') if (os.path.exists('Makefile') or os.path.exists('./unix/Makefile')) and not os.path.exists('.__make_done'): if os.path.exists('Makefile'): if os.path.exists('./configure'): cmd = 'make' else: cmd = 'make PREFIX=%(awelocal)s' % vars() else: cmd = '(cd unix ; make)' print('Making %(dir)s: %(cmd)s' % vars()) dosystem('%(cmd)s > ../%(dir)s_make.log 2> ../%(dir)s_make.err' % vars()) touch('.__make_done') if (os.path.exists('Makefile') or os.path.exists('./unix/Makefile')) and not os.path.exists('.__install_done'): if os.path.exists('Makefile'): if os.path.exists('./configure'): cmd = 'make install' else: cmd = 'make install PREFIX=%(awelocal)s' % vars() else: cmd ='(cd unix; make install)' print('Local installing %(dir)s: %(cmd)s' % vars()) dosystem('%(cmd)s > ../%(dir)s_install.log 2> ../%(dir)s_install.err' % vars()) touch('.__install_done') os.chdir(savecwd) dellist = [] for v in os.environ.keys(): if v != 'AWEHOME' and v.startswith('AWE'): dellist.append(v) for v in dellist: del(os.environ[v]) if 'ORACLE_HOME' in os.environ: oraclehome = os.environ['ORACLE_HOME'] else: oraclehome = '' # Directories in awehome for dir in versions+[awearch]: awdir = os.path.join(awehome, dir) if not os.path.exists(awdir): os.makedirs(awdir) # Directories in awehome/awearch for dir in versions+[instantclientdir, 'local', 'build']: awdir = os.path.join(awehome, awearch, dir) if not os.path.exists(awdir): os.makedirs(awdir) awebase = os.path.join(awehome, awearch) awelocal = os.path.join(awehome, awearch, 'local') awebuild = os.path.join(awehome, awearch, 'build') instantclientdir = os.path.join(awebase, instantclientdir) for dir in versions: awdir = os.path.join(awebuild, dir + pythonver) if not os.path.exists(awdir): os.makedirs(awdir) awe = os.path.join(awelocal, 'bin', 'awe' + pythonver) cwd = os.getcwd() os.chdir(awebuild) blog = open('.build_log', 'a') blog.write('Date: %s Host: %s PID: %d Args: %s AWEHOME: %s\n' % (time.ctime(), socket.getfqdn(), os.getpid(), sys.argv, awehome)) blog.close() realawetarget = awetarget awetarget = realawetarget.split('/')[1] realtargets = ['omegacen/install'] if not usepip: realtargets.append('omegacen/common') if realawetarget not in realtargets: realtargets.append(realawetarget) codetargets = ['omegacen/install', 'omegacen/common'] if realawetarget not in codetargets: codetargets.append(realawetarget) newpythonexe = os.path.join(awelocal, 'bin', 'python' + pythonver) restartawesetup(newpythonexe, startdir, thisthing) cvsenv = '' if rs == 'cvs': if cvsroot == defaultcvsroot: cvs_passfile = os.path.join(awehome, '.cvspass') checkcvspass(cvs_passfile) cvsenv = 'CVS_PASSFILE=%(cvs_passfile)s' % vars() elif cvsroot.startswith(':pserver:'): cvs_passfile = os.path.join(os.environ['HOME'], '.cvspass') cvsenv = 'CVS_PASSFILE=%(cvs_passfile)s' % vars() else: cvsenv = 'CVS_RSH=ssh' wantcvspassword = False if not targetcvsroot: targetcvsroot = cvsroot if targetcvsroot == defaultcvsroot: cvs_passfile = os.path.join(awehome, '.cvspass') checkcvspass(cvs_passfile) targetcvsenv = 'CVS_PASSFILE=%(cvs_passfile)s' % vars() elif targetcvsroot.startswith(':pserver:'): cvs_passfile = os.path.join(awehome, '.cvspass') wantcvspassword = True if os.path.exists(cvs_passfile): cvsfd = open(cvs_passfile, 'r') line = cvsfd.readline() while wantcvspassword and line: wantcvspassword = not targetcvsroot in line line = cvsfd.readline() cvsfd.close() targetcvsenv = 'CVS_PASSFILE=%(cvs_passfile)s' % vars() else: targetcvsenv = 'CVS_RSH=ssh' for realtarget in codetargets: target = realtarget.split('/')[1] if rs == 'cvs': if target in ['install', 'common']: cvs_env = cvsenv if target == 'install': cvs_module = '-f ' + target else: cvs_module = 'awe/'+target cvs_password_needed = False cvs_root = cvsroot else: cvs_env = targetcvsenv cvs_module = 'awe/'+target cvs_password_needed = wantcvspassword cvs_root = targetcvsroot else: gitenv = git_path for aweversion in versions: awecodeversion = os.path.join(awehome, aweversion) if not os.path.exists(os.path.join(awecodeversion, target)): os.chdir(awecodeversion) log = os.path.join(awebuild, rs+'install-%(target)s-%(aweversion)s.log' % vars()) err = os.path.join(awebuild, rs+'install-%(target)s-%(aweversion)s.err' % vars()) print('Getting "%(aweversion)s" version of %(target)s code' % vars()) if rs == 'cvs': if find_in_path('cvs'): if cvs_password_needed: dosystem('env %(cvs_env)s HOME=%(awehome)s cvs -d %(cvs_root)s login' % vars()) wantcvspassword = False if aweversion == 'current': dosystem('env %(cvs_env)s HOME=%(awehome)s cvs -q -d %(cvs_root)s checkout -d %(target)s %(cvs_module)s > %(log)s 2> %(err)s' % vars()) else: dosystem('env %(cvs_env)s HOME=%(awehome)s cvs -q -d %(cvs_root)s checkout -r %(aweversion)s -d %(target)s %(cvs_module)s > %(log)s 2> %(err)s' % vars()) else: print('WARNING: cvs missing, please install') else: if awetarget in gitprivatekeytargets and not git_passfile and gitlab == defaultgitlab: passfile = os.path.join(awecodeversion, '.gitpass') renamepf = True checkgitpass(passfile) else: passfile = git_passfile renamepf = False if passfile and os.path.exists(passfile): lines = docommand('%(git_path)s --version' % vars()) if lines[0][:-1].split()[2].startswith('1.'): raise Exception('git versionj should be 2.0 or higher [= %s]' % (lines[0][:-1].split()[2])) else: gitenv = 'GIT_SSH_COMMAND="ssh -i %(passfile)s" %(git_path)s' % vars() else: gitenv = git_path print('env %(gitenv)s clone --progress %(gitlab)s:%(realtarget)s.git -b %(aweversion)s %(target)s > %(log)s 2> %(err)s' % vars()) dosystem('env %(gitenv)s clone --progress %(gitlab)s:%(realtarget)s.git -b %(aweversion)s %(target)s > %(log)s 2> %(err)s' % vars()) if renamepf: if os.path.exists(os.path.join(awecodeversion, target, '.gitprvssh.key')): os.remove(passfile) else: os.rename(passfile, os.path.join(awecodeversion, target, '.gitpass')) if not os.path.exists(os.path.join(awecodeversion, target)): tarball = 'awetxt-%(target)s-%(aweversion)s.tgz' % vars() print('Failed to get "%(aweversion)s" version of %(target)s code via %(rs)s! Downloading tarball %(tarball)s!' % vars()) os.chdir(awehome) dodownload(downloadurl+'/'+tarball, tarball) if not os.path.exists(tarball): print('FATAL: Cannot find tarball %(tarball)s @ %(downloadurl)s!' % vars()) sys.exit(1) untar(tarball) os.remove(tarball) os.chdir(awebuild) checkawesetupversion(awecodeversion) elif os.path.exists(os.path.join(awecodeversion, target)): usepassfile = False passfilemode = 0 os.chdir(os.path.join(awecodeversion, target)) log = os.path.join(awebuild, rs+'update-%(target)s-%(aweversion)s.log' % vars()) err = os.path.join(awebuild, rs+'update-%(target)s-%(aweversion)s.err' % vars()) print('Updating "%(aweversion)s" version of %(target)s code' % vars()) if rs == 'cvs': if find_in_path('cvs'): if aweversion == 'current': dosystem('env %(cvs_env)s HOME=%(awehome)s cvs -q update -dPA > %(log)s 2> %(err)s' % vars()) else: dosystem('env %(cvs_env)s HOME=%(awehome)s cvs -q update -r%(aweversion)s -dP > %(log)s 2> %(err)s' % vars()) else: print('WARNING: cvs missing, please install') else: if target in gitprivatekeytargets and not git_passfile and gitlab == defaultgitlab: passfile = os.path.join(awecodeversion, target, '.gitprvssh.key') if not os.path.exists(passfile): passfile = os.path.join(awecodeversion, target, '.gitpass') else: passfile = git_passfile if passfile and os.path.exists(passfile): lines = docommand('%(git_path)s --version' % vars()) if lines[0][:-1].split()[2].startswith('1.'): raise Exception('git versionj should be 2.0 or higher [= %s]' % (lines[0][:-1].split()[2])) else: usepassfile = True gitenv = 'GIT_SSH_COMMAND="ssh -i %(passfile)s" %(git_path)s' % vars() else: gitenv = git_path if usepassfile: passfilemode = os.stat(passfile).st_mode if passfilemode & (stat.S_IRWXO | stat.S_IRWXG): os.chmod(passfile, stat.S_IXUSR | stat.S_IRUSR) else: passfilemode = 0 dosystem('env %(gitenv)s pull --progress %(gitlab)s:%(realtarget)s.git %(target)s > %(log)s 2> %(err)s' % vars()) if usepassfile and passfilemode: os.chmod(passfile, passfilemode) os.chdir(awebuild) checkawesetupversion(awecodeversion) if 'all' in skiplist: sys.exit(0) sitecustomize = os.path.join(awehome, versions[0], 'install', 'util', 'sitecustomize.py') # For SuSe 12.* or higher we need --libdir=%(awelocal)s/lib or a link from /lib64 to /lib!! susefix = awearch.startswith('Linux-SuSE-') and float(awearch.split('-')[2]) >= 12.0 if not 'base' in skiplist: if not os.path.exists(os.path.basename(pythonurl)): dodownload(pythonurl) if not os.path.exists(pythondir): untar(os.path.basename(pythonurl)) for i, pythonpatchurl in zip(range(len(pythonpatchurls)),pythonpatchurls): if not os.path.exists(os.path.basename(pythonpatchurl)): dodownload(pythonpatchurl) if not os.path.exists(os.path.join(pythondir, '.__patch_done_%d' % i)): dosystem('patch -p0 < %s' % os.path.basename(pythonpatchurl)) touch(os.path.join(pythondir, '.__patch_done_%d' % i)) if not os.path.exists(os.path.join(pythondir, '.__configure_done')): os.chdir(pythondir) cmd = './configure --prefix=%(awelocal)s' % vars() installopenssl = (pythonver in ['3.10'] and ( awearch.startswith('Linux-centos-') or awearch.startswith('Linux-redhat-')) and docommand2('openssl version')[0].split()[1].startswith('1.0')) if installopenssl: install_local(opensslurl, awelocal, commands=['./config --prefix=%(awelocal)s' % vars(), 'make', 'make install']) cmd = cmd + ' --with-openssl=%(awelocal)s --with-openssl-rpath=auto' % vars() if pythondir.startswith('Python-2.7.') and int(pythondir.split('.')[2])>=9: cmd += ' --with-ensurepip=install' if susefix: cmd += ' --libdir=%(awelocal)s/lib' % vars() if os.path.exists('/usr/local/include'): cmd = 'env C_INCLUDE_PATH=/usr/include ' + cmd if awearch.startswith('Darwin-'): # For macos? cmd = cmd + ' --enable-shared' gnudir = os.uname()[4] + '-linux-gnu' if os.path.exists(os.path.join('/usr/lib', gnudir)) and os.path.exists(os.path.join('/usr/include', gnudir)): cmd = 'env LDFLAGS="-L %s" CPPFLAGS="-I %s" ' % (os.path.join('/usr/lib', gnudir), os.path.join('/usr/include', gnudir)) + cmd print('Configuring %(pythondir)s: %(cmd)s' % vars()) dosystem('%(cmd)s > ../%(pythondir)s_configure.log 2> ../%(pythondir)s_configure.err' % vars()) os.chdir(awebuild) touch(os.path.join(pythondir, '.__configure_done')) if not os.path.exists(os.path.join(pythondir, '.__make_done')): os.chdir(pythondir) cmd = 'make' % vars() if os.path.exists('/usr/local/include'): cmd = 'env C_INCLUDE_PATH=/usr/include ' + cmd if awearch.startswith('Darwin-'): # For macos? cmd = 'env CPPFLAGS=-DWITH_NEXT_FRAMEWORK=1 ' + cmd print('Making %(pythondir)s: %(cmd)s' %vars()) dosystem('%(cmd)s > ../%(pythondir)s_make.log 2> ../%(pythondir)s_make.err' % vars()) count = 0 ldcount = 0 if not testpythonmodule('zlib'): # zlib print('Python module zlib not found! Trying local install!') count += 1 install_local(zliburl, awelocal, config_opts='--shared') ldcount += 1 if not testpythonmodule('bz2'): # bzip2 print('Python module bz2 not found! Trying local install!') count += 1 install_local(bzip2url, awelocal) install_local(bzip2url, awelocal, commands=['make clean', 'make -f Makefile-libbz2_so PREFIX=%(awelocal)s' % vars(), 'cp libbz2.so.1.0.6 %(awelocal)s/lib/' % vars(), 'rm -f %(awelocal)s/lib/libbz2.a' % vars(), '(cd %(awelocal)s/lib;ln -s libbz2.so.1.0.6 libbz2.so.1.0;ln -s libbz2.so.1.0.6 libbz2.so)' % vars()]) ldcount += 1 if not testpythonmodule('readline'): # readline print('Python module readline not found!') count += 1 install_local(readlineurl, awelocal) ldcount += 1 if (pythonver.startswith('2.') and not testpythonmodule('Tkinter')) or (not pythonver.startswith('2.') and not testpythonmodule('tkinter')): if pythonver.startswith('2.'): print('Python module Tkinter not found!') else: print('Python module tkinter not found!') count += 1 os.rename('setup.py', 'setup.py.original') fd_i = open('setup.py.original', 'r') fd_o = open('setup.py', 'w') line = fd_i.readline() copy = 1 while line: if copy and line.startswith(' # Check for BLT extension'): copy = 0 elif not copy and line.startswith(' # Add the Tcl/Tk libraries'): copy = 1 if copy: fd_o.write(line) else: fd_o.write('#AWE' + line) line = fd_i.readline() fd_i.close() fd_o.close() if pythonver == '2.5': if not testpythonmodule('readline'): # readline print('Python module readline not found!') count += 1 install_local(readlineurl, awelocal) ldcount += 1 if not testpythonmodule('tkinter') and 1: # not yet tested print('Python module tkinter not found!') # count += 1 # os.chdir(awebuild) # dodownload(tclurl) # untar(os.path.basename(tclurl)) # dodownload(tkurl) # untar(os.path.basename(tkurl)) # os.chdir(pythondir) # install_local(tclurl, awelocal) # install_local(tkurl, awelocal) if count: cmd = './configure --prefix=%(awelocal)s' % vars() if pythondir.startswith('Python-2.7.') and int(pythondir.split('.')[2])>=9: cmd += ' --with-ensurepip=install' if susefix: cmd += ' --libdir=%(awelocal)s/lib' % vars() if os.path.exists('/usr/local/include'): cmd = 'env C_INCLUDE_PATH=/usr/include ' + cmd if awearch.startswith('Darwin-'): # For macos? cmd = cmd + ' --enable-shared' gnudir = os.uname()[4] + '-linux-gnu' if os.path.exists(os.path.join('/usr/lib', gnudir)) and os.path.exists(os.path.join('/usr/include', gnudir)): cmd = 'env LDFLAGS="-L %s" CPPFLAGS="-I %s" ' % (os.path.join('/usr/lib', gnudir), os.path.join('/usr/include', gnudir)) + cmd print('Configuring %(pythondir)s [try 2]: %(cmd)s' %vars()) dosystem('%(cmd)s > ../%(pythondir)s_configure_2.log 2> ../%(pythondir)s_configure_2.err' % vars()) cmd = 'make' if awearch.startswith('Darwin-'): # For macos? cmd = 'env CPPFLAGS=-DWITH_NEXT_FRAMEWORK=1 ' + cmd if os.path.exists('/usr/local/include'): cmd = 'env C_INCLUDE_PATH=/usr/include ' + cmd if ldcount: cmd = 'env LD_RUN_PATH=%(awelocal)s/lib %(cmd)s' % vars() print('Making %(pythondir)s [try 2]: %(cmd)s' %vars()) dosystem('%(cmd)s > ../%(pythondir)s_make.log_2 2> ../%(pythondir)s_make_2.err' % vars()) if pythonver == '2.5': pexe = './python.exe' if not os.path.exists(pexe): pexe = './python' pycode = '''\ import readline print hasattr(readline,"set_pre_input_hook") ''' r = docommand('env -uPYTHONVERBOSE %(pexe)s' % vars(), pycode)[0] if r[:-1] == 'False': os.rename('Modules/readline.c', 'Modules/readline.c.old') fi = open('Modules/readline.c.old', 'r') fo = open('Modules/readline.c', 'w') fo.write('#define HAVE_RL_PRE_INPUT_HOOK 1\n') line = fi.readline() while line: fo.write(line) line = fi.readline() fi.close() fo.close() cmd = 'make' print('Making %(pythondir)s [try 3]: %(cmd)s' %vars()) dosystem('%(cmd)s > ../%(pythondir)s_make.log_3 2> ../%(pythondir)s_make_3.err' % vars()) os.chdir(awebuild) touch(os.path.join(pythondir, '.__make_done')) if not os.path.exists(os.path.join(pythondir, '.__install_done')): os.chdir(pythondir) cmd = 'make install' print('Installing %(pythondir)s: %(cmd)s' % vars()) dosystem('%(cmd)s > ../%(pythondir)s_install.log 2> ../%(pythondir)s_install.err' % vars()) os.chdir(awebuild) touch(os.path.join(pythondir, '.__install_done')) print('Installing done!') removelink = False if not os.path.exists(os.path.join(awelocal, 'bin', 'awe' + pythonver)): if usepip: removelink = True os.chdir(os.path.join(awelocal, 'bin')) os.symlink('python' + pythonver, 'awe' + pythonver) os.chdir(awebuild) else: os.chdir(os.path.join(awelocal, 'bin')) if not os.path.exists(os.path.join(awelocal, 'bin', 'awe')): os.symlink('python' + pythonver, 'awe') os.symlink('python' + pythonver, 'awe' + pythonver) os.chdir(awebuild) dositecustomize(awehome, awearch, sitecustomize, pythonver, awetarget, versions[0]) # check whether awe can find the awearch aweversion = versions[0] if (ipv[0] > 3) or (ipv[0] == 3 and ipv[1] >= 8): pipcmd = os.path.join(awelocal, 'bin', 'pip' + pythonver) lines = docommand('env -uPYTHONVERBOSE AWEHOME=%(awehome)s AWEVERSION=%(aweversion)s %(pipcmd)s install distro' % vars()) for line in lines[:-1]: print(line[:-1]) def getmyarch(awelocal, pythonver): import tempfile pytexe = os.path.join(awelocal, 'bin', 'python' + pythonver) aweexe = os.path.join(awelocal, 'bin', 'awe' + pythonver) (fd, tempfilename) = tempfile.mkstemp(suffix='.py') os.close(fd) fd = open(tempfilename, 'w') fd.write('''\ import os import sys fd = open(sys.argv[0][:-2] + 'data', 'w') fd.write(os.environ['AWEARCH'] + '\\n') fd.close() os.remove(sys.argv[0]) sys.exit(1) ''') fd.close() pid = os.fork() if not pid: os.execv(pytexe, [aweexe, tempfilename]) os.wait() time.sleep(1) fd = open(tempfilename[:-2] + 'data', 'r') myawearch = fd.readline()[:-1] fd.close() os.remove(tempfilename[:-2] + 'data') return myawearch myawearch = getmyarch(awelocal, pythonver) if myawearch != awearch: print(''' AWEARCH not matching the AWEARCH calculated by %(awe)s! You should run awesetup with awearch=%(myawearch)s ''' % vars()) if not awearchoverrule: sys.exit(1) elif awearchoverrule: awearchoverrule = False if removelink: os.remove(awe) if not oraclehome and not (os.path.exists(os.path.join(instantclientdir, 'libclntsh.so')) or os.path.exists(os.path.join(instantclientdir, 'libclntsh.dylib'))): if not os.path.exists(instantclientdir): os.makedirs(instantclientdir) for instantclienturl in instantclienturls: if not os.path.exists(os.path.basename(instantclienturl)): dodownload(instantclienturl) unzip(os.path.basename(instantclienturl), instantclientdir) os.chdir(instantclientdir) if os.path.exists('libclntsh.so.11.1') and not os.path.exists('libclntsh.so'): os.symlink('libclntsh.so.11.1', 'libclntsh.so') if os.path.exists('libclntsh.dylib.10.1') and not os.path.exists('libclntsh.dylib'): os.symlink('libclntsh.dylib.10.1', 'libclntsh.dylib') #if os.path.exists('libclntsh.dylib.10.1'): subprocess.call(['install_name_tool', '-id', os.path.join(instantclientdir, 'libclntsh.dylib.10.1'), 'libclntsh.dylib.10.1']) os.chdir(awebuild) restartawesetup(newpythonexe, startdir, thisthing) if usepip == 1: pip = os.path.join(awehome, awearch, 'local', 'bin', 'pip2') dosystem('%(pip)s install --isolated --no-cache-dir --upgrade pip virtualenv==16.3.0' % vars()) elif usepip == 2: pip = os.path.join(awehome, awearch, 'local', 'bin', 'pip3') #dosystem('%(pip)s install --isolated --no-cache-dir --upgrade "pip%(wp)s"' % vars()) # if usepip: skiplist.append('common') aweexe = {} for realtarget in realtargets: target = realtarget.split('/')[1] if target in skiplist + ['install']: continue for aweversion in versions: if usepip == 0: awetomaticpath = os.path.join(awehome, aweversion, 'install', target, 'awetomatic.py') if os.path.exists(awetomaticpath): os.chdir(os.path.join(awebuild, aweversion + pythonver)) print('Running %(target)s awetomatic for AWEVERSION=%(aweversion)s' % vars()) if susefix: libpath = os.path.join(awehome, awearch, aweversion, target, 'lib') lib64path = os.path.join(awehome, awearch, aweversion, target, 'lib64') if not os.path.exists(libpath): os.makedirs(libpath) if not os.path.exists(lib64path): os.symlink(libpath, lib64path) dosystem('env %(cvsenv)s HOME=%(awehome)s AWEHOME=%(awehome)s AWEVERSION=%(aweversion)s AWETARGET=%(awetarget)s %(awe)s %(awetomaticpath)s > awetomatic-%(target)s.sh' % vars()) print('Running generated awetomatic-script "awetomatic-%(target)s.sh" for AWEVERSION=%(aweversion)s' % vars()) dosystem('env %(cvsenv)s HOME=%(awehome)s AWEHOME=%(awehome)s AWEVERSION=%(aweversion)s AWETARGET=%(awetarget)s sh awetomatic-%(target)s.sh' % vars()) os.chdir(awebuild) else: print('awetomatic not found: %(awetomaticpath)s' % vars()) else: movedprevious = True oldvenvcfg = os.path.join(awehome, awearch, aweversion, target, 'pyvenv.cfg') oldvenvexe = os.path.join(awehome, awearch, aweversion, target, 'bin', 'python%d' % (usepip+1)) oldpythonversion = '' if os.path.exists(oldvenvcfg): oldpythonversion = [line[:-1] for line in open(oldvenvcfg, 'r').readlines() if line.startswith('version')][0].split()[-1] elif os.path.exists(oldvenvexe): oldpythonversion = docommand(oldvenvexe + ' -V')[0].split()[-1] if oldpythonversion and (oldpythonversion != pythonversion): print('Moving old pyvenv out of the way! "%s" -> "%s"' % (os.path.join(awehome, awearch, aweversion, target), os.path.join(awehome, awearch, aweversion, target + '-' + oldpythonversion))) os.rename(os.path.join(awehome, awearch, aweversion, target), os.path.join(awehome, awearch, aweversion, target + '-' + oldpythonversion)) os.chdir(os.path.join(awebuild, aweversion + pythonver)) aweexeversion = os.path.join(awehome, awearch, aweversion, target, 'bin', 'awe') aweexe[aweversion] = aweexeversion # We need the path to the compiled version in .../local/bin pythonpath = os.path.join(awehome, awearch, 'local', 'bin', 'python%d' % (usepip+1)) print('Running virtualawe for "%(target)s", version = "%(aweversion)s", with "%(pythonpath)s"' % vars()) PATH = os.environ['PATH'] HOME = os.environ['HOME'] dosystem('env -i PATH=$PATH HOME=$HOME AWEVERSION=%(aweversion)s AWETARGET=%(target)s AWEARCH=%(awearch)s %(pythonpath)s %(thisthing)s exename=virtualawe %(aweversion)s %(target)s %(instantclientdir)s' % vars()) checkawelinks(awehome, awearch, aweversion, awetarget, pythonver) awetomaticpath = os.path.join(awehome, aweversion, 'install', target, 'awetomatic.py') if os.path.exists(awetomaticpath): print('Running %(target)s awetomatic for AWEVERSION=%(aweversion)s' % vars()) dosystem('env %(cvsenv)s HOME=%(awehome)s AWEHOME=%(awehome)s AWEVERSION=%(aweversion)s AWETARGET=%(awetarget)s AWEARCH=%(awearch)s %(aweexeversion)s -m install.%(target)s.awetomatic > awetomatic-%(target)s.sh' % vars()) print('Running generated awetomatic-script "awetomatic-%(target)s.sh" for AWEVERSION=%(aweversion)s AWEARCH=%(awearch)s' % vars()) dosystem('env %(cvsenv)s HOME=%(awehome)s AWEHOME=%(awehome)s AWEVERSION=%(aweversion)s AWETARGET=%(awetarget)s AWEARCH=%(awearch)s sh awetomatic-%(target)s.sh' % vars()) if not os.path.exists(os.path.join(awelocal, 'bin', 'awe' + pythonver)): if usepip: path2python = find_in_path('python') if not path2python: path2python = '/usr/bin/python' os.chdir(os.path.join(awelocal, 'bin')) open('awe' + pythonver, 'w').write('''\ #!%s from sys import argv from os import environ, path, execve pv = argv[0][argv[0].rindex("awe")+3:] exepath = path.join(environ.get("AWEHOME", "%s"), environ.get("AWEARCH", "%s"), environ.get("AWEVERSION", "%s"), environ.get("AWETARGET", "%s"), 'bin', 'awe'+pv) argv[0] = exepath if not path.exists(exepath): exepath = path.join(environ.get("AWEHOME", "%s"), environ.get("AWEARCH", "%s"), 'local', 'bin', 'python'+pv) execve(exepath, argv, environ) ''' % (path2python, awehome, awearch, versions[0], awetarget, awehome, awearch)) os.chmod('awe' + pythonver, stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH) os.chdir(awebuild) if not os.path.exists(os.path.join(awelocal, 'bin', 'awe')): os.chdir(os.path.join(awelocal, 'bin')) os.symlink('awe' + pythonver, 'awe') os.chdir(awebuild) os.chdir(cwd) awecmdprefix = '' if awearchoverrule: awecmdprefix = 'env AWEARCH=%(awearch)s ' % vars() awecmds = awe if usepip: awecmdslist = [] updatecmdslist = [] for v in versions: awecmdslist.append('%(awecmdprefix)s%(awehome)s/%(awearch)s/%(v)s/%(awetarget)s/bin/awe%(pythonver)s' % (vars())) updatecmdslist.append('%(awecmdprefix)s%(awehome)s/%(awearch)s/%(v)s/%(awetarget)s/bin/awe%(pythonver)s %(awehome)s/%(v)s/common/toolbox/update_%(rs)s.py' % (vars())) awecmds = '\nand : '.join(awecmdslist) updatecmds = '\nand : '.join(updatecmdslist) else: updatecmdslist = [] for v in versions: updatecmdslist.append('%(awecmdprefix)s%(awe)s %(awehome)s/%(v)s/common/toolbox/update_%(rs)s.py' % (vars())) updatecmds = '\nand : '.join(updatecmdslist) print('''\ Finished installing Python-%(pythonversion)s! Your awe executable is : %(awecmds)s The build logs (*.log and *.err) can be found starting here: %(awebuild)s You can update and compile the python code by : %(updatecmds)s Questions, ask Willem-Jan Vriend : wjvriend@astro-wise.org ''' % vars()) def grid_awe(awehome, awearch, awetarget, versions, aweexecutable): for v in versions: awelocal = os.path.join(awehome, awearch, 'local') awelocalbin = os.path.join(awelocal, 'bin') aweexe = os.path.join(awehome, awearch, 'local', 'bin', aweexecutable) pythonexe = os.path.join(awehome, awearch, 'local', 'bin', 'python' + aweexecutable[3:]) cmd = 'env -uPYTHONVERBOSE AWEHOME=%(awehome)s AWEVERSION=%(v)s ' % vars() + pythonexe + ' -V' exactpythonversion = docommand(cmd)[-1].split()[-1] ipv = [int(vp) for vp in exactpythonversion.split('.')] pythonver = '%d.%d' % (ipv[0], ipv[1]) #'%s.%s' % tuple(exactpythonversion.split('.')[:2]) usepip = ipv[0] > 2 or ipv[2] > 10 if usepip: # check whether all paths are correct checkawelinks(awehome, awearch, v, awetarget, pythonver) sitecustomize = os.path.join(awehome, v, 'install', 'util', 'sitecustomize.py') if not os.path.exists(sitecustomize): sitecustomize = os.path.join(awehome, v, 'install', awetarget, 'sitecustomize.py') if not os.path.exists(sitecustomize): sitecustomize = os.path.join(awehome, v, awetarget, 'toolbox', 'sitecustomize.py') dositecustomize(awehome, awearch, sitecustomize, pythonver, awetarget, force=True) for d in os.listdir(os.path.join(awehome, awearch)): if d.startswith('instantclient'): libdir = os.path.join(awehome, awearch, d) aweshell = os.path.join(awehome, 'gawe') fd = open(aweshell, 'w') fd.write('''\ #!/usr/bin/env python from sys import argv from os import environ, path, execve pv = "%(pythonver)s" awehome="%(awehome)s" aweversion="%(v)s" awearch="%(awearch)s" awetarget="%(awetarget)s" environ["AWEARCH"] = awearch environ["AWEHOME"] = awehome environ["AWETARGET"] = awetarget environ["AWEVERSION"] = aweversion environ["database_engine"] = 'oracle_11g' environ["LD_LIBRARY_PATH"] = path.join(awehome, awearch, 'instantclient_11_2') environ["log_dir"] = "" environ["project"] = "" environ["PYTHONHOME"] = path.join(awehome, awearch, 'local') environ["PYTHONSTARTUP"] = "" exepath = path.join(awehome, awearch, aweversion, awetarget, 'bin', 'awe'+pv) if not path.exists(exepath): exepath = path.join(awehome, awearch, 'local', 'bin', 'awe'+pv) if path.exists(exepath): argv[0] = exepath execve(exepath, argv, environ) print('"%%(exepath)s" not present!' %% vars()) ''' % vars()) # fd.write('#!/bin/sh\n') # fd.write('ver="${AWEVERSION:-%(v)s}"\n' % vars()) # fd.write('exec env PYTHONSTARTUP= log_dir= project= database_engine=oracle_11g LD_LIBRARY_PATH=%(libdir)s PYTHONHOME=%(awelocal)s AWEVERSION=${ver} %(aweexe)s $@\n' % vars()) fd.close() if not os.access(aweshell, os.X_OK): os.chmod(aweshell, stat.S_IRWXU) def grid_install_awe(*kw_list, **kw_dict): if 'AWEHOME' in os.environ: awehome = os.environ['AWEHOME'] else: awehome = os.path.join(os.getcwd(), 'awehome') awehome.rstrip(os.path.sep) if 'AWETARGET' in os.environ: awetarget = os.environ['AWETARGET'] else: awetarget = 'common' if 'awearch' in kw_dict: awearch = kw_dict['awearch'] else: if 'awehome' in kw_dict: awearch = whichos(awehome=kw_dict['awehome']) else: awearch = whichos(awehome=awehome) argdefaults = { 'awehome': awehome, 'aweexecutable': 'awe', 'awetarget': awetarget, 'mode': 'install', 'versions': 'master,develop', 'uploadurl': driveurl+'/awesoft/grid-install', 'downloadurl': driveurl+'/awesoft/grid-install', 'awearch': awearch, } argcount = 0 for arg in argdefaults.keys(): if arg in kw_dict.keys(): val = kw_dict[arg] argcount += 1 else: val = argdefaults[arg] if arg == 'awehome': awehome = val awehome.rstrip(os.path.sep) elif arg == 'aweexecutable': aweexecutable = val elif arg == 'awetarget': awetarget = val elif arg == 'versions': versions = [] for v in val.split(','): if v not in versions: versions.append(v) elif arg == 'uploadurl': uploadurl = val.rstrip('/') elif arg == 'downloadurl': downloadurl = val.rstrip('/') elif arg == 'mode': mode = val elif arg == 'awearch': awearch = val else: raise Exception('Something wrong with %(arg)s [%(val)s]' % vars()) pyver = aweexecutable[aweexecutable.rindex('awe')+3:] # version of python needed if pyver and pyver[:2] == '2.': pyver = '' # starting with python 3 we put the pythonversion in the bin tarball if kw_list or argcount != len(kw_dict): keyorder = ['awehome', 'aweexecutable', 'awetarget', 'versions', 'uploadurl', 'downloadurl', 'mode', 'awearch'] print('''\ Name: grid-install-awe Purpose: To install a precompiled version of astro-wise software (for grid computing). Usage: %s %s= %s= %s= %s= %s= %s= %s= %s= root of install AWEHOME [%s] awe executable (+ python version) [%s] wanted AWETARGET distribution i.e. common, astro, lofar [%s] versions [%s] uploadurl [%s] downloadurl [%s] grid mode (install, setup, uploadonly) [%s] awearch [ %s] Version: %s ''' % tuple([sys.argv[0]] + keyorder + [argdefaults[k] for k in keyorder] + __version__.split()[1:2])) sys.exit(0) cwd = os.getcwd() if mode == 'install': if not os.path.exists(awehome): os.makedirs(awehome) for v in versions: if pyver: tbname = 'awebin-%(awetarget)s-%(v)s-%(pyver)s-%(awearch)s.tgz' % vars() else: tbname = 'awebin-%(awetarget)s-%(v)s-%(awearch)s.tgz' % vars() tburl = '%(downloadurl)s/%(tbname)s' % vars() dodownload(tburl) untar(tbname, path=awehome) os.remove(tbname) for v in versions: tbname = 'awetxt-%(awetarget)s-%(v)s.tgz' % vars() tburl = '%(downloadurl)s/%(tbname)s' % vars() dodownload(tburl) untar(tbname, path=awehome) os.remove(tbname) grid_awe(awehome, awearch, awetarget, versions, aweexecutable) elif mode == 'setup': awesetup(awehome=awehome, awearch=awearch, awetarget=awetarget, downloadurl=downloadurl, versions=','.join(versions)) makebintarball(awehome=awehome, awearch=awearch, awetarget=awetarget, versions=versions, uploadurl=uploadurl) elif mode == 'uploadonly': makebintarball(awehome=awehome, awearch=awearch, awetarget=awetarget, versions=versions, uploadurl=uploadurl) else: print('mode=%(mode)s unknown!' % vars()) def virtualawe(*kw_list, **kw_dict): from pdb import set_trace as breakpoint, pm import os import pip import sys import distutils.sysconfig oldawehome = os.environ['AWEHOME'] awearch = os.environ['AWEARCH'] aweversion, awetarget, instantclientdir = kw_list sys.path.insert(0, os.path.join(oldawehome, aweversion)) from common.util.targetplatform import targetplatform destdir = os.path.join(oldawehome, targetplatform(), aweversion, awetarget) #instantclientdir = os.path.join(oldawehome, targetplatform(), 'instantclient_11_2') bindir = os.path.join(destdir, 'bin') pipcmd = os.path.join(bindir, 'pip') wantedpipversion = wantedpip # Check whether distro module is needed ipv = [int(n) for n in sys.version.split()[0].split('.')] distroneeded = ipv[0] > 3 or ipv[0] == 3 and ipv[1] >= 8 assert '/' not in aweversion assert '/' not in awetarget if sys.version_info.major == 2: import virtualenv virtualenv.create_environment(destdir, no_pip=False, symlink=True) if pip.__version__.startswith('7.'): dosystem('%(pipcmd)s2 install --isolated --no-cache-dir --upgrade pip virtualenv' % vars()) elif sys.version_info.major == 3: import venv venv.create(destdir, with_pip=True, symlinks=True) dosystem('%(pipcmd)s3 install --isolated --no-cache-dir --upgrade "pip%(wantedpipversion)s"' % vars()) else: raise NotImplementedError("Python {0}".format(sys.version_info.major)) aweecustomize = '''\ # This file was generated by {} import os os.environ['AWETARGET'] = '{}' os.environ['AWEVERSION'] = '{}' os.environ['AWEARCH'] = '{}' del(os) try: import matplotlib matplotlib.rcParams['image.origin'] = 'lower' matplotlib.rcParams['interactive'] = True del(matplotlib) except: pass '''.format(__file__, awetarget, aweversion, awearch) open(os.path.join(distutils.sysconfig.get_python_lib(False,False,destdir),'awecustomize.py'),'w').write(aweecustomize) if not os.path.exists(os.path.join(bindir, 'awe')): os.symlink('python', os.path.join(bindir, 'awe')) #requirements = 'http://drive.astro-wise.org/awesoft/awex/{version.major}.{version.minor}/{aweversion}/{awetarget}/requirements.txt'.format(version=sys.version_info,aweversion=aweversion,awetarget=awetarget) requirements = os.path.join(oldawehome, aweversion, 'install', awetarget, 'requirements.txt') env = dict( ORACLE_HOME = instantclientdir, LD_RUN_PATH = instantclientdir, ) if hasattr(pip, 'utils'): env = {'ORACLE_HOME': instantclientdir, 'LD_RUN_PATH': instantclientdir} pip.utils.call_subprocess([pipcmd, 'install', '--disable-pip-version-check', '--no-cache-dir', '-r', requirements], extra_environ=env) if distroneeded: pip.utils.call_subprocess([pipcmd, 'install', '--disable-pip-version-check', '--no-cache-dir', 'distro'], extra_environ=env) else: dosystem('env ORACLE_HOME=%s LD_RUN_PATH=%s %s install --disable-pip-version-check --no-cache-dir -r %s' % (env['ORACLE_HOME'], env['LD_RUN_PATH'], pipcmd, requirements)) if distroneeded: dosystem('env ORACLE_HOME=%s LD_RUN_PATH=%s %s install --disable-pip-version-check --no-cache-dir distro' % (env['ORACLE_HOME'], env['LD_RUN_PATH'], pipcmd)) if __name__ == '__main__': kw_dict = {} kw_list = [] exename = os.path.basename(sys.argv[0]) for arg in sys.argv[1:]: if '=' in arg: isp = arg.index('=') if isp: if arg[:isp] == 'exename': exename = arg[isp+1:] else: kw_dict[arg[:isp]] = arg[isp+1:] else: kw_list.append(arg) else: kw_list.append(arg) if exename.startswith('awesetup'): awesetup(*kw_list, **kw_dict) elif exename.startswith('grid-install-awe'): grid_install_awe(*kw_list, **kw_dict) elif exename == 'virtualawe': virtualawe(*kw_list, **kw_dict) elif exename.startswith('whichos'): print(whichos(*kw_list, **kw_dict))