3 files changed, 159 insertions, 0 deletions
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100755
index 000000000..8db5bb0bf
--- /dev/null
+++ b/scripts/__init__.py
@@ -0,0 +1,17 @@
+#! /usr/bin/env python
+
+#   Copyright 2015 WebAssembly Community Group participants
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# Empty __init__.py file: Python treats the directory as containing a package.
diff --git a/scripts/storage.py b/scripts/storage.py
new file mode 100755
index 000000000..8ae2e2676
--- /dev/null
+++ b/scripts/storage.py
@@ -0,0 +1,53 @@
+#! /usr/bin/env python
+
+#   Copyright 2016 WebAssembly Community Group participants
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import glob
+import json
+import os
+import urllib2
+
+
+STORAGE_BASE = 'https://storage.googleapis.com/wasm-llvm/builds/git/'
+
+
+def download_revision(force_latest):
+  name = 'latest' if force_latest else 'lkgr'
+  downloaded = urllib2.urlopen(STORAGE_BASE + name).read().strip()
+  # TODO: for now try opening as JSON, if that doesn't work then the content is
+  #       just a hash. The waterfall is in the process of migrating to JSON.
+  info = None
+  try:
+    info = json.loads(downloaded)
+  except:
+    pass
+  return info['build'] if type(info) == dict else downloaded
+
+
+def download_tar(tar_pattern, directory, revision):
+  tar_path = os.path.join(directory, tar_pattern)
+  revision_tar_path = tar_path % revision
+  if os.path.isfile(revision_tar_path):
+    print 'Already have `%s`' % revision_tar_path
+  else:
+    print 'Downloading `%s`' % revision_tar_path
+    with open(revision_tar_path, 'w+') as f:
+      f.write(urllib2.urlopen(STORAGE_BASE + tar_pattern % revision).read())
+  # Remove any previous tarfiles.
+  for older_tar in glob.glob(tar_path % '*'):
+    if older_tar != revision_tar_path:
+      print 'Removing older tar file `%s`' % older_tar
+      os.remove(older_tar)
+  return revision_tar_path
diff --git a/scripts/support.py b/scripts/support.py
new file mode 100755
index 000000000..a99fb7604
--- /dev/null
+++ b/scripts/support.py
@@ -0,0 +1,89 @@
+#! /usr/bin/env python
+
+#   Copyright 2016 WebAssembly Community Group participants
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import filecmp
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+
+def _open_archive(tarfile, tmp_dir):
+  with tempfile.TemporaryFile(mode='w+') as f:
+    try:
+      subprocess.check_call(['tar', '-xvf', tarfile], cwd=tmp_dir, stdout=f)
+    except:
+      f.seek(0)
+      sys.stderr.write(f.read())
+      raise
+  return os.listdir(tmp_dir)
+
+
+def _files_same(dir1, dir2, basenames):
+  diff = filecmp.cmpfiles(dir1, dir2, basenames)
+  return 0 == len(diff[1] + diff[2])
+
+
+def _dirs_same(dir1, dir2, basenames):
+  for d in basenames:
+    left = os.path.join(dir1, d)
+    right = os.path.join(dir2, d)
+    if not (os.path.isdir(left) and os.path.isdir(right)):
+      return False
+    diff = filecmp.dircmp(right, right)
+    if 0 != len(diff.left_only + diff.right_only + diff.diff_files +
+                diff.common_funny + diff.funny_files):
+      return False
+  return True
+
+
+def _move_files(dirfrom, dirto, basenames):
+  for f in basenames:
+    from_file = os.path.join(dirfrom, f)
+    to_file = os.path.join(dirto, f)
+    if os.path.isfile(to_file):
+      os.path.remove(to_file)
+    shutil.move(from_file, to_file)
+
+
+def _move_dirs(dirfrom, dirto, basenames):
+    for d in basenames:
+      from_dir = os.path.join(dirfrom, d)
+      to_dir = os.path.join(dirto, d)
+      if os.path.isdir(to_dir):
+        shutil.rmtree(to_dir)
+      shutil.move(from_dir, to_dir)
+
+
+def untar(tarfile, outdir):
+  """Returns True if untar content differs from pre-existing outdir content."""
+  tmpdir = tempfile.mkdtemp()
+  try:
+    untared = _open_archive(tarfile, tmpdir)
+    files = [f for f in untared if os.path.isfile(os.path.join(tmpdir, f))]
+    dirs = [d for d in untared if os.path.isdir(os.path.join(tmpdir, d))]
+    assert len(files) + len(dirs) == len(untared), 'Only files and directories'
+    if _files_same(tmpdir, outdir, files) and _dirs_same(tmpdir, outdir, dirs):
+      # Nothing new or different in the tarfile.
+      return False
+    # Some or all of the files / directories are new.
+    _move_files(tmpdir, outdir, files)
+    _move_dirs(tmpdir, outdir, dirs)
+    return True
+  finally:
+    if os.path.isdir(tmpdir):
+      shutil.rmtree(tmpdir)