More hacking on the snapshot system.

Author: graydon

Makefile.in

@@ -108,7 +108,6 @@ GENERATED :=
 %:: s.%
 %:: SCCS/s.%
 
-
 ######################################################################
 # Standard library variables
 ######################################################################
@@ -135,6 +134,13 @@ SREQ1 := stage1/rustc$(X) $(LREQ) stage2/glue.o stage2/$(CFG_STDLIB)
 SREQ2 := stage2/rustc$(X) $(LREQ) stage3/glue.o stage3/$(CFG_STDLIB)
 
 
+######################################################################
+# Exports for sub-utilities
+######################################################################
+
+export CFG_SRC_DIR
+
+
 ######################################################################
 # Single-target rules
 ######################################################################
@@ -179,5 +185,6 @@ include $(CFG_SRC_DIR)/mk/rustllvm.mk
 include $(CFG_SRC_DIR)/mk/docs.mk
 include $(CFG_SRC_DIR)/mk/tests.mk
 include $(CFG_SRC_DIR)/mk/dist.mk
+include $(CFG_SRC_DIR)/mk/snap.mk
 include $(CFG_SRC_DIR)/mk/clean.mk
 include $(CFG_SRC_DIR)/mk/autodep.mk

src/etc/get-snapshot.py

@@ -9,34 +9,15 @@ def snap_filename_hash_part(snap):
     raise Exception("unable to find hash in filename: " + snap)
   return match.group(1)
 
-def get_snapshot_and_check_hash(snap):
-
-  hsh = snap_filename_hash_part(snap)
-
-  h = hashlib.sha1()
-  url = download_url_base + "/" + snap
-  print "downloading " + url
-  u = urllib2.urlopen(url)
-  print "checking hash on download"
-  data = u.read()
-  h.update(data)
-  if h.hexdigest() != hsh:
-    raise Exception("hash check failed on " + snap)
-
-  print "hash ok"
-  with open(os.path.join(download_dir_base, snap), "w+b") as f:
-    f.write(data)
-  return True
-
 def unpack_snapshot(snap):
   dl_path = os.path.join(download_dir_base, snap)
-  print "opening snapshot " + dl_path
+  print("opening snapshot " + dl_path)
   tar = tarfile.open(dl_path)
   kernel = get_kernel()
   for name in snapshot_files[kernel]:
     p = os.path.join("rust-stage0", name)
     fp = os.path.join("stage0", name)
-    print "extracting " + fp
+    print("extracting " + fp)
     tar.extract(p, download_unpack_base)
     tp = os.path.join(download_unpack_base, p)
     shutil.move(tp, fp)
@@ -80,16 +61,16 @@ def determine_last_snapshot_for_platform():
 # Main
 
 snap = determine_last_snapshot_for_platform()
-print "determined most recent snapshot: " + snap
 dl = os.path.join(download_dir_base, snap)
-if (os.path.exists(dl)):
-  if (snap_filename_hash_part(snap) == hash_file(dl)):
-    print "found existing download with ok hash"
-  else:
-    print "bad hash on existing download, re-fetching"
-    get_snapshot_and_check_hash(snap)
+url = download_url_base + "/" + snap
+print("determined most recent snapshot: " + snap)
+
+if (not os.path.exists(dl)):
+  get_url_to_file(url, dl)
+
+if (snap_filename_hash_part(snap) == hash_file(dl)):
+  print("got download with ok hash")
 else:
-  print "no cached download, fetching"
-  get_snapshot_and_check_hash(snap)
+  raise Exception("bad hash on download")
 
 unpack_snapshot(snap)

src/etc/make-snapshot.py

@@ -21,4 +21,4 @@
 
 shutil.move(file0, file1)
 
-print file1
+print(file1)

src/etc/snapshot.py

@@ -1,6 +1,10 @@
-import re, os, sys, hashlib, tarfile, shutil, subprocess, urllib2, tempfile
+import re, os, sys, hashlib, tarfile, shutil, subprocess, tempfile
 
-snapshotfile = "snapshots.txt"
+src_dir = os.getenv("CFG_SRC_DIR")
+if not src_dir:
+  raise Exception("missing env var CFG_SRC_DIR")
+
+snapshotfile = os.path.join(src_dir, "snapshots.txt")
 download_url_base = "http://dl.rust-lang.org/stage0-snapshots"
 download_dir_base = "dl"
 download_unpack_base = os.path.join(download_dir_base, "unpack")
@@ -59,11 +63,16 @@ def get_cpu():
 def get_platform():
   return "%s-%s" % (get_kernel(), get_cpu())
 
+def scrub(b):
+  if sys.version_info >= (3,) and type(b) == bytes:
+    return b.decode('ascii')
+  else:
+    return b
 
 def cmd_out(cmdline):
     p = subprocess.Popen(cmdline,
                          stdout=subprocess.PIPE)
-    return p.communicate()[0].strip()
+    return scrub(p.communicate()[0].strip())
 
 
 def local_rev_info(field):
@@ -82,8 +91,10 @@ def local_rev_short_sha():
 def local_rev_committer_date():
     return local_rev_info("ci")
 
+def get_url_to_file(u,f):
+  subprocess.check_call(["curl", "-o", f, u])
 
 def hash_file(x):
     h = hashlib.sha1()
-    h.update(open(x).read())
-    return h.hexdigest()
+    h.update(open(x, "rb").read())
+    return scrub(h.hexdigest())

src/snapshots.txt

@@ -1,3 +1,6 @@
+S 2011-05-02 ed40c85
+  winnt-i386 e69c11fbc62639ac3a3eef7ea36c9ad77209e2b1
+
 S 2011-04-29 7b95b5c
   linux-i386 f0e166816ce34adc9f7202bd3cfbd80623505f28
   macos-i386 abf2ee279da63676ca17c9dc9e54d04d8f752b00