# HG changeset patch
# User Doug Simon <doug.simon@oracle.com>
# Date 1347285031 -7200
# Node ID f45d5cb03c3b6ffec8b9377341ae13660afe3e15
# Parent  744223b9ce38961e47baa18cabe3ef6b1305c339
added support for downloading dependencies that are directories

diff -r 744223b9ce38 -r f45d5cb03c3b mxtool/mx.py
--- a/mxtool/mx.py	Mon Sep 10 15:24:29 2012 +0200
+++ b/mxtool/mx.py	Mon Sep 10 15:50:31 2012 +0200
@@ -321,6 +321,9 @@
         self.mustExist = mustExist
         self.sourcePath = sourcePath
         self.sourceUrls = sourceUrls
+        for url in urls:
+            if url.endswith('/') != self.path.endswith(os.sep):
+                abort('Path for dependency directory must have a URL ending with "/": path=' + self.path + ' url=' + url)
 
     def get_path(self, resolve):
         path = self.path
@@ -1103,18 +1106,19 @@
     # Try it with the Java tool first since it can show a progress counter
     myDir = dirname(__file__)
 
-    javaSource = join(myDir, 'URLConnectionDownload.java')
-    javaClass = join(myDir, 'URLConnectionDownload.class')
-    if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource):
-        subprocess.check_call([java().javac, '-d', myDir, javaSource])
-    if run([java().java, '-cp', myDir, 'URLConnectionDownload', path] + urls) == 0:
-        return
+    if not path.endswith(os.sep):
+        javaSource = join(myDir, 'URLConnectionDownload.java')
+        javaClass = join(myDir, 'URLConnectionDownload.class')
+        if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource):
+            subprocess.check_call([java().javac, '-d', myDir, javaSource])
+        if run([java().java, '-cp', myDir, 'URLConnectionDownload', path] + urls) == 0:
+            return
 
     def url_open(url):
         userAgent = 'Mozilla/5.0 (compatible)'
         headers = { 'User-Agent' : userAgent }
         req = urllib2.Request(url, headers=headers)
-        return urllib2.urlopen(req);
+        return urllib2.urlopen(req)
 
     for url in urls:
         try:
@@ -1136,8 +1140,18 @@
             else:
                 with contextlib.closing(url_open(url)) as f:
                     data = f.read()
-                with open(path, 'wb') as f:
-                    f.write(data)
+                if path.endswith(os.sep):
+                    # Scrape directory listing for relative URLs
+                    hrefs = re.findall(r' href="([^"]*)"', data)
+                    if len(hrefs) != 0:
+                        for href in hrefs:
+                            if not '/' in href:
+                                download(join(path, href), [url + href], verbose)
+                    else:
+                        log('no locals hrefs scraped from ' + url)
+                else:
+                    with open(path, 'wb') as f:
+                        f.write(data)
             return
         except IOError as e:
             log('Error reading from ' + url + ': ' + str(e))