beverley-gorry/colmap-vslamlab

1# Copyright (c), ETH Zurich and UNC Chapel Hill.2# All rights reserved.3#4# Redistribution and use in source and binary forms, with or without5# modification, are permitted provided that the following conditions are met:6#7#     * Redistributions of source code must retain the above copyright8#       notice, this list of conditions and the following disclaimer.9#10#     * Redistributions in binary form must reproduce the above copyright11#       notice, this list of conditions and the following disclaimer in the12#       documentation and/or other materials provided with the distribution.13#14#     * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of15#       its contributors may be used to endorse or promote products derived16#       from this software without specific prior written permission.17#18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"19# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE20# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE21# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE22# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR23# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF24# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS25# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN26# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)27# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE28# POSSIBILITY OF SUCH DAMAGE.29 30 31import argparse32import datetime33import multiprocessing34import os35import socket36import time37import urllib38import xml.etree.ElementTree as ElementTree39 40import urllib241import urlparse42 43PER_PAGE = 50044SORT = "date-posted-desc"45URL = (46    "https://api.flickr.com/services/rest/?method=flickr.photos.search&"47    "api_key=%s&text=%s&sort=%s&per_page=%d&page=%d&min_upload_date=%s&"48    "max_upload_date=%s&format=rest&extras=url_o,url_l,url_c,url_z,url_n"49)50MAX_PAGE_REQUESTS = 551MAX_PAGE_TIMEOUT = 2052MAX_IMAGE_REQUESTS = 353TIME_SKIP = 24 * 60 * 6054MAX_DATE = time.time()55MIN_DATE = MAX_DATE - TIME_SKIP56 57 58def parse_args():59    parser = argparse.ArgumentParser()60    parser.add_argument("--search_text", required=True)61    parser.add_argument("--api_key", required=True)62    parser.add_argument("--image_path", required=True)63    parser.add_argument("--num_procs", type=int, default=10)64    parser.add_argument("--max_days_without_image", type=int, default=365)65    args = parser.parse_args()66    return args67 68 69def compose_url(page, api_key, text, min_date, max_date):70    return URL % (71        api_key,72        text,73        SORT,74        PER_PAGE,75        page,76        str(min_date),77        str(max_date),78    )79 80 81def parse_page(page, api_key, text, min_date, max_date):82    f = None83    for _ in range(MAX_PAGE_REQUESTS):84        try:85            f = urllib2.urlopen(86                compose_url(page, api_key, text, min_date, max_date),87                timeout=MAX_PAGE_TIMEOUT,88            )89        except socket.timeout:90            continue91        else:92            break93 94    if f is None:95        return {96            "pages": "0",97            "total": "0",98            "page": "0",99            "perpage": "0",100        }, tuple()101 102    response = f.read()103    root = ElementTree.fromstring(response)104 105    if root.attrib["stat"] != "ok":106        raise IOError107 108    photos = []109    for photo in root.iter("photo"):110        photos.append(photo.attrib)111 112    return root.find("photos").attrib, photos113 114 115class PhotoDownloader(object):116    def __init__(self, image_path):117        self.image_path = image_path118 119    def __call__(self, photo):120        # Find the URL corresponding to the highest image resolution. We will121        # need this URL here to determine the image extension (typically .jpg,122        # but could be .png, .gif, etc).123        url = None124        for url_suffix in ("o", "l", "k", "h", "b", "c", "z"):125            url_attr = "url_%s" % url_suffix126            if photo.get(url_attr) is not None:127                url = photo.get(url_attr)128                break129 130        if url is not None:131            # Note that the following statement may fail in Python 3. urlparse132            # may need to be replaced with urllib.parse.133            url_filename = urlparse.urlparse(url).path134            image_ext = os.path.splitext(url_filename)[1]135 136            image_name = "%s_%s%s" % (photo["id"], photo["secret"], image_ext)137            path = os.path.join(self.image_path, image_name)138            if not os.path.exists(path):139                print(url)140                for _ in range(MAX_IMAGE_REQUESTS):141                    try:142                        urllib.urlretrieve(url, path)143                    except urllib.ContentTooShortError:144                        continue145                    else:146                        break147 148 149def main():150    args = parse_args()151 152    downloader = PhotoDownloader(args.image_path)153    pool = multiprocessing.Pool(processes=args.num_procs)154 155    num_pages = float("inf")156    page = 0157 158    min_date = MIN_DATE159    max_date = MAX_DATE160 161    days_in_row = 0162 163    search_text = args.search_text.replace(" ", "-")164 165    while num_pages > page:166        page += 1167 168        metadata, photos = parse_page(169            page, args.api_key, search_text, min_date, max_date170        )171 172        num_pages = int(metadata["pages"])173 174        print(78 * "=")175        print("Page:\t\t", page, "of", num_pages)176        print("Min-Date:\t", datetime.datetime.fromtimestamp(min_date))177        print("Max-Date:\t", datetime.datetime.fromtimestamp(max_date))178        print("Num-Photos:\t", len(photos))179        print(78 * "=")180 181        try:182            pool.map_async(downloader, photos).get(1e10)183        except KeyboardInterrupt:184            pool.wait()185            break186 187        if page >= num_pages:188            max_date -= TIME_SKIP189            min_date -= TIME_SKIP190            page = 0191 192        if num_pages == 0:193            days_in_row = days_in_row + 1194            num_pages = float("inf")195 196            print("    No images in", days_in_row, "days in a row")197 198            if days_in_row == args.max_days_without_image:199                break200        else:201            days_in_row = 0202 203 204if __name__ == "__main__":205    main()206