diff options
| -rw-r--r-- | .travis.yml | 2 | ||||
| -rwxr-xr-x | scripts/__init__.py | 17 | ||||
| -rwxr-xr-x | scripts/storage.py | 49 | ||||
| -rwxr-xr-x | scripts/support.py | 50 | ||||
| -rwxr-xr-x | update.py | 69 |
5 files changed, 123 insertions, 64 deletions
diff --git a/.travis.yml b/.travis.yml index 94ee9512b..e09463a85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,7 @@ install: before_script: # Check the style of a subset of Python code until the other code is updated. - - flake8 update.py + - flake8 update.py ./scripts/* script: - cmake . -DCMAKE_C_FLAGS="$COMPILER_FLAGS" -DCMAKE_CXX_FLAGS="$COMPILER_FLAGS" diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100755 index 000000000..8db5bb0bf --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,17 @@ +#! /usr/bin/env python + +# Copyright 2015 WebAssembly Community Group participants +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Empty __init__.py file: Python treats the directory as containing a package. diff --git a/scripts/storage.py b/scripts/storage.py new file mode 100755 index 000000000..b15c76855 --- /dev/null +++ b/scripts/storage.py @@ -0,0 +1,49 @@ +#! /usr/bin/env python + +# Copyright 2016 WebAssembly Community Group participants +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +import urllib2 + + +STORAGE_BASE = 'https://storage.googleapis.com/wasm-llvm/builds/git/' + + +def download_revision(force_latest): + name = 'latest' if force_latest else 'lkgr' + downloaded = urllib2.urlopen(STORAGE_BASE + name).read().strip() + # TODO: for now try opening as JSON, if that doesn't work then the content is + # just a hash. The waterfall is in the process of migrating to JSON. + info = None + try: + info = json.loads(downloaded) + except: + pass + return info['build'] if type(info) == dict else downloaded + + +def download_tar(tar_pattern, directory, revision): + tar_path = os.path.join(directory, tar_pattern) + revision_tar_path = tar_path % revision + if not os.path.isfile(revision_tar_path): + with open(revision_tar_path, 'w+') as f: + f.write(urllib2.urlopen(STORAGE_BASE + tar_pattern % revision).read()) + # Remove any previous tarfiles. + for older_tar in glob.glob(tar_path): + if older_tar != revision_tar_path: + os.path.remove(older_tar) + return revision_tar_path diff --git a/scripts/support.py b/scripts/support.py new file mode 100755 index 000000000..95d663fe0 --- /dev/null +++ b/scripts/support.py @@ -0,0 +1,50 @@ +#! /usr/bin/env python + +# Copyright 2016 WebAssembly Community Group participants +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import filecmp +import os +import shutil +import subprocess +import sys +import tempfile + + +def untar(tarfile, outdir): + """Returns True if the untar dir differs from a pre-existing dir.""" + tmp_dir = tempfile.mkdtemp() + try: + with tempfile.TemporaryFile(mode='w+') as f: + try: + subprocess.check_call(['tar', '-xvf', tarfile], cwd=tmp_dir, stdout=f) + except: + f.seek(0) + sys.stderr.write(f.read()) + raise + untar_outdir = os.path.join(tmp_dir, os.path.basename(outdir)) + if os.path.exists(outdir): + diff = filecmp.dircmp(untar_outdir, outdir) + if not (diff.left_only + diff.right_only + diff.diff_files + + diff.common_funny + diff.funny_files): + # outdir already existed with exactly the same content. + return False + shutil.rmtree(outdir) + # The untar files are different, or there was no previous outdir. + print 'Updating', outdir + shutil.move(untar_outdir, outdir) + return True + finally: + if os.path.isdir(tmp_dir): + shutil.rmtree(tmp_dir) @@ -14,83 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import filecmp -import glob -import json import os -import shutil import subprocess import sys -import tempfile -import urllib2 + +import scripts.storage +import scripts.support -STORAGE_BASE = 'https://storage.googleapis.com/wasm-llvm/builds/git/' BASE_DIR = os.path.abspath('test') REVISION_PATH = os.path.join(BASE_DIR, 'revision') TORTURE_TAR = 'wasm-torture-s-%s.tbz2' TORTURE_DIR = os.path.join(BASE_DIR, 'torture-s') -def download_revision(force_latest): - name = 'latest' if force_latest else 'lkgr' - downloaded = urllib2.urlopen(STORAGE_BASE + name).read().strip() - # TODO: for now try opening as JSON, if that doesn't work then the content is - # just a hash. The waterfall is in the process of migrating to JSON. - info = None - try: - info = json.loads(downloaded) - except: - pass - return info['build'] if type(info) == dict else downloaded - - def write_revision(revision): with open(REVISION_PATH, 'w') as f: f.write(revision) -def download_tar(tar_pattern, revision): - tar_path = os.path.join(BASE_DIR, tar_pattern) - revision_tar_path = tar_path % revision - if not os.path.isfile(revision_tar_path): - with open(revision_tar_path, 'w+') as f: - f.write(urllib2.urlopen(STORAGE_BASE + tar_pattern % revision).read()) - # Remove any previous tarfiles. - for older_tar in glob.glob(tar_path): - if older_tar != revision_tar_path: - os.path.remove(older_tar) - return revision_tar_path - - -def untar(tarfile, outdir): - """Returns True if the untar dir differs from a pre-existing dir.""" - tmp_dir = tempfile.mkdtemp() - try: - with tempfile.TemporaryFile(mode='w+') as f: - try: - subprocess.check_call(['tar', '-xvf', tarfile], cwd=tmp_dir, stdout=f) - except: - f.seek(0) - sys.stderr.write(f.read()) - raise - untar_outdir = os.path.join(tmp_dir, os.path.basename(outdir)) - if os.path.exists(outdir): - diff = filecmp.dircmp(untar_outdir, outdir) - if not (diff.left_only + diff.right_only + diff.diff_files + - diff.common_funny + diff.funny_files): - # outdir already existed with exactly the same content. - return False - shutil.rmtree(outdir) - # The untar files are different, or there was no previous outdir. - print 'Updating', outdir - shutil.move(untar_outdir, outdir) - return True - finally: - if os.path.isdir(tmp_dir): - shutil.rmtree(tmp_dir) - - def run(force_latest, override_hash): subprocess.check_call(['git', 'submodule', 'sync', '--quiet']) subprocess.check_call(['git', 'submodule', 'init', '--quiet']) @@ -99,8 +41,9 @@ def run(force_latest, override_hash): 'git', 'pull', 'origin', 'master', '--quiet']) updates = 0 revision = (override_hash if override_hash else - download_revision(force_latest=force_latest)) - updates += untar(download_tar(TORTURE_TAR, revision), TORTURE_DIR) + scripts.storage.download_revision(force_latest=force_latest)) + downloaded = scripts.storage.download_tar(TORTURE_TAR, BASE_DIR, revision) + updates += scripts.support.untar(downloaded, TORTURE_DIR) if updates: # Only update revision if the files it downloaded are different. print 'Updating revision to', revision |