1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
|
#! /usr/bin/env python
# Copyright 2015 WebAssembly Community Group participants
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import filecmp
import glob
import json
import os
import shutil
import subprocess
import sys
import tempfile
import urllib2
STORAGE_BASE = 'https://storage.googleapis.com/wasm-llvm/builds/git/'
BASE_DIR = os.path.abspath('test')
REVISION_PATH = os.path.join(BASE_DIR, 'revision')
TORTURE_TAR = 'wasm-torture-s-%s.tbz2'
TORTURE_DIR = os.path.join(BASE_DIR, 'torture-s')
def download_revision(force_latest):
name = 'latest' if force_latest else 'lkgr'
downloaded = urllib2.urlopen(STORAGE_BASE + name).read().strip()
# TODO: for now try opening as JSON, if that doesn't work then the content is
# just a hash. The waterfall is in the process of migrating to JSON.
info = None
try:
info = json.loads(downloaded)
except:
pass
return info['build'] if type(info) == dict else downloaded
def write_revision(revision):
with open(REVISION_PATH, 'w') as f:
f.write(revision)
def download_tar(tar_pattern, revision):
tar_path = os.path.join(BASE_DIR, tar_pattern)
revision_tar_path = tar_path % revision
if not os.path.isfile(revision_tar_path):
with open(revision_tar_path, 'w+') as f:
f.write(urllib2.urlopen(STORAGE_BASE + tar_pattern % revision).read())
# Remove any previous tarfiles.
for older_tar in glob.glob(tar_path):
if older_tar != revision_tar_path:
os.path.remove(older_tar)
return revision_tar_path
def untar(tarfile, outdir):
"""Returns True if the untar dir differs from a pre-existing dir."""
tmp_dir = tempfile.mkdtemp()
try:
with tempfile.TemporaryFile(mode='w+') as f:
try:
subprocess.check_call(['tar', '-xvf', tarfile], cwd=tmp_dir, stdout=f)
except:
f.seek(0)
sys.stderr.write(f.read())
raise
untar_outdir = os.path.join(tmp_dir, os.path.basename(outdir))
if os.path.exists(outdir):
diff = filecmp.dircmp(untar_outdir, outdir)
if not (diff.left_only + diff.right_only + diff.diff_files +
diff.common_funny + diff.funny_files):
# outdir already existed with exactly the same content.
return False
shutil.rmtree(outdir)
# The untar files are different, or there was no previous outdir.
print 'Updating', outdir
shutil.move(untar_outdir, outdir)
return True
finally:
if os.path.isdir(tmp_dir):
shutil.rmtree(tmp_dir)
def run(force_latest, override_hash):
subprocess.check_call(['git', 'submodule', 'sync', '--quiet'])
subprocess.check_call(['git', 'submodule', 'init', '--quiet'])
subprocess.check_call(['git', 'submodule', 'update', '--quiet'])
subprocess.check_call(['git', 'submodule', 'foreach',
'git', 'pull', 'origin', 'master', '--quiet'])
updates = 0
revision = (override_hash if override_hash else
download_revision(force_latest=force_latest))
updates += untar(download_tar(TORTURE_TAR, revision), TORTURE_DIR)
if updates:
# Only update revision if the files it downloaded are different.
print 'Updating revision to', revision
write_revision(revision)
def getargs():
import argparse
parser = argparse.ArgumentParser(
description='Update the repository dependencies.')
parser.add_argument('--force-latest', action='store_true',
help='Sync to latest waterfall build, not lkgr')
parser.add_argument('--override-hash', type=str, default=None,
help='Sync to specific hash from waterfall build')
return parser.parse_args()
if __name__ == '__main__':
args = getargs()
sys.exit(run(force_latest=args.force_latest,
override_hash=args.override_hash))
|
