1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
#!/usr/bin/env python
import os
import glob
import re
import tempfile
import subprocess
from config import *
builds_list = []
remove_prefix = re.compile(remove)
def create_path_list(path, path_to_create):
while len(path) > 0:
path_to_create.insert(0, path)
head, tail = os.path.split(path)
if len(tail.strip()) == 0: # Just in case path ends with a / or \
path = head
head, tail = os.path.split(path)
path = head
return path_to_create
def do_sftp_transfer(build, path_to_create, files):
fd, fpath = tempfile.mkstemp()
fnull = open(os.devnull, 'w')
for path in path_to_create:
os.write(fd, '-mkdir ' + remote_prefix + str(path) + '\n')
for rfile in files:
os.write(fd, '-put ' + str(rfile) + ' ' + remote_prefix +
os.path.dirname(remove_prefix.sub('', str(rfile))) + '\n')
os.close(fd)
subprocess.call(["sftp", "-i", sync_key_file, "-q", "-r", "-b", fpath,
"%s@%s" % (sync_user, host)], stdout = fnull, stderr = fnull)
subprocess.call(["ssh", "-i", move_key_file, "%s@%s" % (move_user, host),
move_cmd, build])
fnull.close()
os.remove(fpath)
def check_remote(job, build_number):
global builds_list
return "%s/%s" % (job, build_number) in builds_list
def load_build_list(fname):
# Command to generate build list:
# find /srv3/snapshots.linaro.org/www/android/ -type d -maxdepth 3
# -mindepth 3 -wholename '*/~*/*' | sed
# 's/\/srv3\/snapshots\.linaro\.org\/www\/android\/~//'
builds_list = []
try:
f = open(fname)
except IOError:
raise
else:
with f:
while 1:
line = f.readline()
if not line:
break
builds_list.append(line.replace("/", "_", 1).strip("\n"))
return builds_list
def move_files(job):
for dirname, dirnames, filenames in os.walk('%s/builds' % job):
for build_number in dirnames:
if os.path.islink(os.path.join(dirname, build_number)):
build_started = "%s/%s" % (job, build_number)
if check_remote(job, build_number):
print "Build skipped: " + build_started
continue
files = [] # List of local files to be transferred
remote_paths = [] # List of full remote paths to be created
path_to_create = [] # List of subpaths of full paths
print "Build started: " + build_started
for src in source_files:
files.extend(glob.glob("%s/archive/%s" %
(os.path.join(dirname, build_number), src)))
for rfile in files:
remote_paths.append(remove_prefix.sub('',
os.path.dirname(rfile)))
uniq_remote_paths = list(set(remote_paths))
for i in uniq_remote_paths:
path_to_create = create_path_list(i, path_to_create)
do_sftp_transfer(build_started, path_to_create, files)
if __name__ == '__main__':
builds_list = load_build_list("./builds-on-snapshots")
cwd = os.getcwd()
os.chdir(jobs_path)
for ejob in jobs_list:
jobs = glob.glob('%s*' % ejob)
for job in jobs:
move_files(job)
os.chdir(cwd)
|
