+++ /dev/null
-from __future__ import absolute_import
-
-import cgi
-import email.utils
-import getpass
-import json
-import logging
-import mimetypes
-import os
-import platform
-import re
-import shutil
-import sys
-import tempfile
-
-try:
- import ssl # noqa
- HAS_TLS = True
-except ImportError:
- HAS_TLS = False
-
-from pip._vendor.six.moves.urllib import parse as urllib_parse
-from pip._vendor.six.moves.urllib import request as urllib_request
-
-import pip
-
-from pip.exceptions import InstallationError, HashMismatch
-from pip.models import PyPI
-from pip.utils import (splitext, rmtree, format_size, display_path,
- backup_dir, ask_path_exists, unpack_file,
- ARCHIVE_EXTENSIONS, consume, call_subprocess)
-from pip.utils.encoding import auto_decode
-from pip.utils.filesystem import check_path_owner
-from pip.utils.logging import indent_log
-from pip.utils.setuptools_build import SETUPTOOLS_SHIM
-from pip.utils.glibc import libc_ver
-from pip.utils.ui import DownloadProgressBar, DownloadProgressSpinner
-from pip.locations import write_delete_marker_file
-from pip.vcs import vcs
-from pip._vendor import requests, six
-from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
-from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth
-from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response
-from pip._vendor.requests.utils import get_netrc_auth
-from pip._vendor.requests.structures import CaseInsensitiveDict
-from pip._vendor.requests.packages import urllib3
-from pip._vendor.cachecontrol import CacheControlAdapter
-from pip._vendor.cachecontrol.caches import FileCache
-from pip._vendor.lockfile import LockError
-from pip._vendor.six.moves import xmlrpc_client
-
-
-__all__ = ['get_file_content',
- 'is_url', 'url_to_path', 'path_to_url',
- 'is_archive_file', 'unpack_vcs_link',
- 'unpack_file_url', 'is_vcs_url', 'is_file_url',
- 'unpack_http_url', 'unpack_url']
-
-
-logger = logging.getLogger(__name__)
-
-
-def user_agent():
- """
- Return a string representing the user agent.
- """
- data = {
- "installer": {"name": "pip", "version": pip.__version__},
- "python": platform.python_version(),
- "implementation": {
- "name": platform.python_implementation(),
- },
- }
-
- if data["implementation"]["name"] == 'CPython':
- data["implementation"]["version"] = platform.python_version()
- elif data["implementation"]["name"] == 'PyPy':
- if sys.pypy_version_info.releaselevel == 'final':
- pypy_version_info = sys.pypy_version_info[:3]
- else:
- pypy_version_info = sys.pypy_version_info
- data["implementation"]["version"] = ".".join(
- [str(x) for x in pypy_version_info]
- )
- elif data["implementation"]["name"] == 'Jython':
- # Complete Guess
- data["implementation"]["version"] = platform.python_version()
- elif data["implementation"]["name"] == 'IronPython':
- # Complete Guess
- data["implementation"]["version"] = platform.python_version()
-
- if sys.platform.startswith("linux"):
- from pip._vendor import distro
- distro_infos = dict(filter(
- lambda x: x[1],
- zip(["name", "version", "id"], distro.linux_distribution()),
- ))
- libc = dict(filter(
- lambda x: x[1],
- zip(["lib", "version"], libc_ver()),
- ))
- if libc:
- distro_infos["libc"] = libc
- if distro_infos:
- data["distro"] = distro_infos
-
- if sys.platform.startswith("darwin") and platform.mac_ver()[0]:
- data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]}
-
- if platform.system():
- data.setdefault("system", {})["name"] = platform.system()
-
- if platform.release():
- data.setdefault("system", {})["release"] = platform.release()
-
- if platform.machine():
- data["cpu"] = platform.machine()
-
- # Python 2.6 doesn't have ssl.OPENSSL_VERSION.
- if HAS_TLS and sys.version_info[:2] > (2, 6):
- data["openssl_version"] = ssl.OPENSSL_VERSION
-
- return "{data[installer][name]}/{data[installer][version]} {json}".format(
- data=data,
- json=json.dumps(data, separators=(",", ":"), sort_keys=True),
- )
-
-
-class MultiDomainBasicAuth(AuthBase):
-
- def __init__(self, prompting=True):
- self.prompting = prompting
- self.passwords = {}
-
- def __call__(self, req):
- parsed = urllib_parse.urlparse(req.url)
-
- # Get the netloc without any embedded credentials
- netloc = parsed.netloc.rsplit("@", 1)[-1]
-
- # Set the url of the request to the url without any credentials
- req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:])
-
- # Use any stored credentials that we have for this netloc
- username, password = self.passwords.get(netloc, (None, None))
-
- # Extract credentials embedded in the url if we have none stored
- if username is None:
- username, password = self.parse_credentials(parsed.netloc)
-
- # Get creds from netrc if we still don't have them
- if username is None and password is None:
- netrc_auth = get_netrc_auth(req.url)
- username, password = netrc_auth if netrc_auth else (None, None)
-
- if username or password:
- # Store the username and password
- self.passwords[netloc] = (username, password)
-
- # Send the basic auth with this request
- req = HTTPBasicAuth(username or "", password or "")(req)
-
- # Attach a hook to handle 401 responses
- req.register_hook("response", self.handle_401)
-
- return req
-
- def handle_401(self, resp, **kwargs):
- # We only care about 401 responses, anything else we want to just
- # pass through the actual response
- if resp.status_code != 401:
- return resp
-
- # We are not able to prompt the user so simply return the response
- if not self.prompting:
- return resp
-
- parsed = urllib_parse.urlparse(resp.url)
-
- # Prompt the user for a new username and password
- username = six.moves.input("User for %s: " % parsed.netloc)
- password = getpass.getpass("Password: ")
-
- # Store the new username and password to use for future requests
- if username or password:
- self.passwords[parsed.netloc] = (username, password)
-
- # Consume content and release the original connection to allow our new
- # request to reuse the same one.
- resp.content
- resp.raw.release_conn()
-
- # Add our new username and password to the request
- req = HTTPBasicAuth(username or "", password or "")(resp.request)
-
- # Send our new request
- new_resp = resp.connection.send(req, **kwargs)
- new_resp.history.append(resp)
-
- return new_resp
-
- def parse_credentials(self, netloc):
- if "@" in netloc:
- userinfo = netloc.rsplit("@", 1)[0]
- if ":" in userinfo:
- return userinfo.split(":", 1)
- return userinfo, None
- return None, None
-
-
-class LocalFSAdapter(BaseAdapter):
-
- def send(self, request, stream=None, timeout=None, verify=None, cert=None,
- proxies=None):
- pathname = url_to_path(request.url)
-
- resp = Response()
- resp.status_code = 200
- resp.url = request.url
-
- try:
- stats = os.stat(pathname)
- except OSError as exc:
- resp.status_code = 404
- resp.raw = exc
- else:
- modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
- content_type = mimetypes.guess_type(pathname)[0] or "text/plain"
- resp.headers = CaseInsensitiveDict({
- "Content-Type": content_type,
- "Content-Length": stats.st_size,
- "Last-Modified": modified,
- })
-
- resp.raw = open(pathname, "rb")
- resp.close = resp.raw.close
-
- return resp
-
- def close(self):
- pass
-
-
-class SafeFileCache(FileCache):
- """
- A file based cache which is safe to use even when the target directory may
- not be accessible or writable.
- """
-
- def __init__(self, *args, **kwargs):
- super(SafeFileCache, self).__init__(*args, **kwargs)
-
- # Check to ensure that the directory containing our cache directory
- # is owned by the user current executing pip. If it does not exist
- # we will check the parent directory until we find one that does exist.
- # If it is not owned by the user executing pip then we will disable
- # the cache and log a warning.
- if not check_path_owner(self.directory):
- logger.warning(
- "The directory '%s' or its parent directory is not owned by "
- "the current user and the cache has been disabled. Please "
- "check the permissions and owner of that directory. If "
- "executing pip with sudo, you may want sudo's -H flag.",
- self.directory,
- )
-
- # Set our directory to None to disable the Cache
- self.directory = None
-
- def get(self, *args, **kwargs):
- # If we don't have a directory, then the cache should be a no-op.
- if self.directory is None:
- return
-
- try:
- return super(SafeFileCache, self).get(*args, **kwargs)
- except (LockError, OSError, IOError):
- # We intentionally silence this error, if we can't access the cache
- # then we can just skip caching and process the request as if
- # caching wasn't enabled.
- pass
-
- def set(self, *args, **kwargs):
- # If we don't have a directory, then the cache should be a no-op.
- if self.directory is None:
- return
-
- try:
- return super(SafeFileCache, self).set(*args, **kwargs)
- except (LockError, OSError, IOError):
- # We intentionally silence this error, if we can't access the cache
- # then we can just skip caching and process the request as if
- # caching wasn't enabled.
- pass
-
- def delete(self, *args, **kwargs):
- # If we don't have a directory, then the cache should be a no-op.
- if self.directory is None:
- return
-
- try:
- return super(SafeFileCache, self).delete(*args, **kwargs)
- except (LockError, OSError, IOError):
- # We intentionally silence this error, if we can't access the cache
- # then we can just skip caching and process the request as if
- # caching wasn't enabled.
- pass
-
-
-class InsecureHTTPAdapter(HTTPAdapter):
-
- def cert_verify(self, conn, url, verify, cert):
- conn.cert_reqs = 'CERT_NONE'
- conn.ca_certs = None
-
-
-class PipSession(requests.Session):
-
- timeout = None
-
- def __init__(self, *args, **kwargs):
- retries = kwargs.pop("retries", 0)
- cache = kwargs.pop("cache", None)
- insecure_hosts = kwargs.pop("insecure_hosts", [])
-
- super(PipSession, self).__init__(*args, **kwargs)
-
- # Attach our User Agent to the request
- self.headers["User-Agent"] = user_agent()
-
- # Attach our Authentication handler to the session
- self.auth = MultiDomainBasicAuth()
-
- # Create our urllib3.Retry instance which will allow us to customize
- # how we handle retries.
- retries = urllib3.Retry(
- # Set the total number of retries that a particular request can
- # have.
- total=retries,
-
- # A 503 error from PyPI typically means that the Fastly -> Origin
- # connection got interrupted in some way. A 503 error in general
- # is typically considered a transient error so we'll go ahead and
- # retry it.
- status_forcelist=[503],
-
- # Add a small amount of back off between failed requests in
- # order to prevent hammering the service.
- backoff_factor=0.25,
- )
-
- # We want to _only_ cache responses on securely fetched origins. We do
- # this because we can't validate the response of an insecurely fetched
- # origin, and we don't want someone to be able to poison the cache and
- # require manual eviction from the cache to fix it.
- if cache:
- secure_adapter = CacheControlAdapter(
- cache=SafeFileCache(cache, use_dir_lock=True),
- max_retries=retries,
- )
- else:
- secure_adapter = HTTPAdapter(max_retries=retries)
-
- # Our Insecure HTTPAdapter disables HTTPS validation. It does not
- # support caching (see above) so we'll use it for all http:// URLs as
- # well as any https:// host that we've marked as ignoring TLS errors
- # for.
- insecure_adapter = InsecureHTTPAdapter(max_retries=retries)
-
- self.mount("https://", secure_adapter)
- self.mount("http://", insecure_adapter)
-
- # Enable file:// urls
- self.mount("file://", LocalFSAdapter())
-
- # We want to use a non-validating adapter for any requests which are
- # deemed insecure.
- for host in insecure_hosts:
- self.mount("https://{0}/".format(host), insecure_adapter)
-
- def request(self, method, url, *args, **kwargs):
- # Allow setting a default timeout on a session
- kwargs.setdefault("timeout", self.timeout)
-
- # Dispatch the actual request
- return super(PipSession, self).request(method, url, *args, **kwargs)
-
-
-def get_file_content(url, comes_from=None, session=None):
- """Gets the content of a file; it may be a filename, file: URL, or
- http: URL. Returns (location, content). Content is unicode."""
- if session is None:
- raise TypeError(
- "get_file_content() missing 1 required keyword argument: 'session'"
- )
-
- match = _scheme_re.search(url)
- if match:
- scheme = match.group(1).lower()
- if (scheme == 'file' and comes_from and
- comes_from.startswith('http')):
- raise InstallationError(
- 'Requirements file %s references URL %s, which is local'
- % (comes_from, url))
- if scheme == 'file':
- path = url.split(':', 1)[1]
- path = path.replace('\\', '/')
- match = _url_slash_drive_re.match(path)
- if match:
- path = match.group(1) + ':' + path.split('|', 1)[1]
- path = urllib_parse.unquote(path)
- if path.startswith('/'):
- path = '/' + path.lstrip('/')
- url = path
- else:
- # FIXME: catch some errors
- resp = session.get(url)
- resp.raise_for_status()
- return resp.url, resp.text
- try:
- with open(url, 'rb') as f:
- content = auto_decode(f.read())
- except IOError as exc:
- raise InstallationError(
- 'Could not open requirements file: %s' % str(exc)
- )
- return url, content
-
-
-_scheme_re = re.compile(r'^(http|https|file):', re.I)
-_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I)
-
-
-def is_url(name):
- """Returns true if the name looks like a URL"""
- if ':' not in name:
- return False
- scheme = name.split(':', 1)[0].lower()
- return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes
-
-
-def url_to_path(url):
- """
- Convert a file: URL to a path.
- """
- assert url.startswith('file:'), (
- "You can only turn file: urls into filenames (not %r)" % url)
-
- _, netloc, path, _, _ = urllib_parse.urlsplit(url)
-
- # if we have a UNC path, prepend UNC share notation
- if netloc:
- netloc = '\\\\' + netloc
-
- path = urllib_request.url2pathname(netloc + path)
- return path
-
-
-def path_to_url(path):
- """
- Convert a path to a file: URL. The path will be made absolute and have
- quoted path parts.
- """
- path = os.path.normpath(os.path.abspath(path))
- url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path))
- return url
-
-
-def is_archive_file(name):
- """Return True if `name` is a considered as an archive file."""
- ext = splitext(name)[1].lower()
- if ext in ARCHIVE_EXTENSIONS:
- return True
- return False
-
-
-def unpack_vcs_link(link, location):
- vcs_backend = _get_used_vcs_backend(link)
- vcs_backend.unpack(location)
-
-
-def _get_used_vcs_backend(link):
- for backend in vcs.backends:
- if link.scheme in backend.schemes:
- vcs_backend = backend(link.url)
- return vcs_backend
-
-
-def is_vcs_url(link):
- return bool(_get_used_vcs_backend(link))
-
-
-def is_file_url(link):
- return link.url.lower().startswith('file:')
-
-
-def is_dir_url(link):
- """Return whether a file:// Link points to a directory.
-
- ``link`` must not have any other scheme but file://. Call is_file_url()
- first.
-
- """
- link_path = url_to_path(link.url_without_fragment)
- return os.path.isdir(link_path)
-
-
-def _progress_indicator(iterable, *args, **kwargs):
- return iterable
-
-
-def _download_url(resp, link, content_file, hashes):
- try:
- total_length = int(resp.headers['content-length'])
- except (ValueError, KeyError, TypeError):
- total_length = 0
-
- cached_resp = getattr(resp, "from_cache", False)
-
- if logger.getEffectiveLevel() > logging.INFO:
- show_progress = False
- elif cached_resp:
- show_progress = False
- elif total_length > (40 * 1000):
- show_progress = True
- elif not total_length:
- show_progress = True
- else:
- show_progress = False
-
- show_url = link.show_url
-
- def resp_read(chunk_size):
- try:
- # Special case for urllib3.
- for chunk in resp.raw.stream(
- chunk_size,
- # We use decode_content=False here because we don't
- # want urllib3 to mess with the raw bytes we get
- # from the server. If we decompress inside of
- # urllib3 then we cannot verify the checksum
- # because the checksum will be of the compressed
- # file. This breakage will only occur if the
- # server adds a Content-Encoding header, which
- # depends on how the server was configured:
- # - Some servers will notice that the file isn't a
- # compressible file and will leave the file alone
- # and with an empty Content-Encoding
- # - Some servers will notice that the file is
- # already compressed and will leave the file
- # alone and will add a Content-Encoding: gzip
- # header
- # - Some servers won't notice anything at all and
- # will take a file that's already been compressed
- # and compress it again and set the
- # Content-Encoding: gzip header
- #
- # By setting this not to decode automatically we
- # hope to eliminate problems with the second case.
- decode_content=False):
- yield chunk
- except AttributeError:
- # Standard file-like object.
- while True:
- chunk = resp.raw.read(chunk_size)
- if not chunk:
- break
- yield chunk
-
- def written_chunks(chunks):
- for chunk in chunks:
- content_file.write(chunk)
- yield chunk
-
- progress_indicator = _progress_indicator
-
- if link.netloc == PyPI.netloc:
- url = show_url
- else:
- url = link.url_without_fragment
-
- if show_progress: # We don't show progress on cached responses
- if total_length:
- logger.info("Downloading %s (%s)", url, format_size(total_length))
- progress_indicator = DownloadProgressBar(max=total_length).iter
- else:
- logger.info("Downloading %s", url)
- progress_indicator = DownloadProgressSpinner().iter
- elif cached_resp:
- logger.info("Using cached %s", url)
- else:
- logger.info("Downloading %s", url)
-
- logger.debug('Downloading from URL %s', link)
-
- downloaded_chunks = written_chunks(
- progress_indicator(
- resp_read(CONTENT_CHUNK_SIZE),
- CONTENT_CHUNK_SIZE
- )
- )
- if hashes:
- hashes.check_against_chunks(downloaded_chunks)
- else:
- consume(downloaded_chunks)
-
-
-def _copy_file(filename, location, link):
- copy = True
- download_location = os.path.join(location, link.filename)
- if os.path.exists(download_location):
- response = ask_path_exists(
- 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)abort' %
- display_path(download_location), ('i', 'w', 'b', 'a'))
- if response == 'i':
- copy = False
- elif response == 'w':
- logger.warning('Deleting %s', display_path(download_location))
- os.remove(download_location)
- elif response == 'b':
- dest_file = backup_dir(download_location)
- logger.warning(
- 'Backing up %s to %s',
- display_path(download_location),
- display_path(dest_file),
- )
- shutil.move(download_location, dest_file)
- elif response == 'a':
- sys.exit(-1)
- if copy:
- shutil.copy(filename, download_location)
- logger.info('Saved %s', display_path(download_location))
-
-
-def unpack_http_url(link, location, download_dir=None,
- session=None, hashes=None):
- if session is None:
- raise TypeError(
- "unpack_http_url() missing 1 required keyword argument: 'session'"
- )
-
- temp_dir = tempfile.mkdtemp('-unpack', 'pip-')
-
- # If a download dir is specified, is the file already downloaded there?
- already_downloaded_path = None
- if download_dir:
- already_downloaded_path = _check_download_dir(link,
- download_dir,
- hashes)
-
- if already_downloaded_path:
- from_path = already_downloaded_path
- content_type = mimetypes.guess_type(from_path)[0]
- else:
- # let's download to a tmp dir
- from_path, content_type = _download_http_url(link,
- session,
- temp_dir,
- hashes)
-
- # unpack the archive to the build dir location. even when only downloading
- # archives, they have to be unpacked to parse dependencies
- unpack_file(from_path, location, content_type, link)
-
- # a download dir is specified; let's copy the archive there
- if download_dir and not already_downloaded_path:
- _copy_file(from_path, download_dir, link)
-
- if not already_downloaded_path:
- os.unlink(from_path)
- rmtree(temp_dir)
-
-
-def unpack_file_url(link, location, download_dir=None, hashes=None):
- """Unpack link into location.
-
- If download_dir is provided and link points to a file, make a copy
- of the link file inside download_dir.
- """
- link_path = url_to_path(link.url_without_fragment)
-
- # If it's a url to a local directory
- if is_dir_url(link):
- if os.path.isdir(location):
- rmtree(location)
- shutil.copytree(link_path, location, symlinks=True)
- if download_dir:
- logger.info('Link is a directory, ignoring download_dir')
- return
-
- # If --require-hashes is off, `hashes` is either empty, the
- # link's embedded hash, or MissingHashes; it is required to
- # match. If --require-hashes is on, we are satisfied by any
- # hash in `hashes` matching: a URL-based or an option-based
- # one; no internet-sourced hash will be in `hashes`.
- if hashes:
- hashes.check_against_path(link_path)
-
- # If a download dir is specified, is the file already there and valid?
- already_downloaded_path = None
- if download_dir:
- already_downloaded_path = _check_download_dir(link,
- download_dir,
- hashes)
-
- if already_downloaded_path:
- from_path = already_downloaded_path
- else:
- from_path = link_path
-
- content_type = mimetypes.guess_type(from_path)[0]
-
- # unpack the archive to the build dir location. even when only downloading
- # archives, they have to be unpacked to parse dependencies
- unpack_file(from_path, location, content_type, link)
-
- # a download dir is specified and not already downloaded
- if download_dir and not already_downloaded_path:
- _copy_file(from_path, download_dir, link)
-
-
-def _copy_dist_from_dir(link_path, location):
- """Copy distribution files in `link_path` to `location`.
-
- Invoked when user requests to install a local directory. E.g.:
-
- pip install .
- pip install ~/dev/git-repos/python-prompt-toolkit
-
- """
-
- # Note: This is currently VERY SLOW if you have a lot of data in the
- # directory, because it copies everything with `shutil.copytree`.
- # What it should really do is build an sdist and install that.
- # See https://github.com/pypa/pip/issues/2195
-
- if os.path.isdir(location):
- rmtree(location)
-
- # build an sdist
- setup_py = 'setup.py'
- sdist_args = [sys.executable]
- sdist_args.append('-c')
- sdist_args.append(SETUPTOOLS_SHIM % setup_py)
- sdist_args.append('sdist')
- sdist_args += ['--dist-dir', location]
- logger.info('Running setup.py sdist for %s', link_path)
-
- with indent_log():
- call_subprocess(sdist_args, cwd=link_path, show_stdout=False)
-
- # unpack sdist into `location`
- sdist = os.path.join(location, os.listdir(location)[0])
- logger.info('Unpacking sdist %s into %s', sdist, location)
- unpack_file(sdist, location, content_type=None, link=None)
-
-
-class PipXmlrpcTransport(xmlrpc_client.Transport):
- """Provide a `xmlrpclib.Transport` implementation via a `PipSession`
- object.
- """
-
- def __init__(self, index_url, session, use_datetime=False):
- xmlrpc_client.Transport.__init__(self, use_datetime)
- index_parts = urllib_parse.urlparse(index_url)
- self._scheme = index_parts.scheme
- self._session = session
-
- def request(self, host, handler, request_body, verbose=False):
- parts = (self._scheme, host, handler, None, None, None)
- url = urllib_parse.urlunparse(parts)
- try:
- headers = {'Content-Type': 'text/xml'}
- response = self._session.post(url, data=request_body,
- headers=headers, stream=True)
- response.raise_for_status()
- self.verbose = verbose
- return self.parse_response(response.raw)
- except requests.HTTPError as exc:
- logger.critical(
- "HTTP error %s while getting %s",
- exc.response.status_code, url,
- )
- raise
-
-
-def unpack_url(link, location, download_dir=None,
- only_download=False, session=None, hashes=None):
- """Unpack link.
- If link is a VCS link:
- if only_download, export into download_dir and ignore location
- else unpack into location
- for other types of link:
- - unpack into location
- - if download_dir, copy the file into download_dir
- - if only_download, mark location for deletion
-
- :param hashes: A Hashes object, one of whose embedded hashes must match,
- or HashMismatch will be raised. If the Hashes is empty, no matches are
- required, and unhashable types of requirements (like VCS ones, which
- would ordinarily raise HashUnsupported) are allowed.
- """
- # non-editable vcs urls
- if is_vcs_url(link):
- unpack_vcs_link(link, location)
-
- # file urls
- elif is_file_url(link):
- unpack_file_url(link, location, download_dir, hashes=hashes)
-
- # http urls
- else:
- if session is None:
- session = PipSession()
-
- unpack_http_url(
- link,
- location,
- download_dir,
- session,
- hashes=hashes
- )
- if only_download:
- write_delete_marker_file(location)
-
-
-def _download_http_url(link, session, temp_dir, hashes):
- """Download link url into temp_dir using provided session"""
- target_url = link.url.split('#', 1)[0]
- try:
- resp = session.get(
- target_url,
- # We use Accept-Encoding: identity here because requests
- # defaults to accepting compressed responses. This breaks in
- # a variety of ways depending on how the server is configured.
- # - Some servers will notice that the file isn't a compressible
- # file and will leave the file alone and with an empty
- # Content-Encoding
- # - Some servers will notice that the file is already
- # compressed and will leave the file alone and will add a
- # Content-Encoding: gzip header
- # - Some servers won't notice anything at all and will take
- # a file that's already been compressed and compress it again
- # and set the Content-Encoding: gzip header
- # By setting this to request only the identity encoding We're
- # hoping to eliminate the third case. Hopefully there does not
- # exist a server which when given a file will notice it is
- # already compressed and that you're not asking for a
- # compressed file and will then decompress it before sending
- # because if that's the case I don't think it'll ever be
- # possible to make this work.
- headers={"Accept-Encoding": "identity"},
- stream=True,
- )
- resp.raise_for_status()
- except requests.HTTPError as exc:
- logger.critical(
- "HTTP error %s while getting %s", exc.response.status_code, link,
- )
- raise
-
- content_type = resp.headers.get('content-type', '')
- filename = link.filename # fallback
- # Have a look at the Content-Disposition header for a better guess
- content_disposition = resp.headers.get('content-disposition')
- if content_disposition:
- type, params = cgi.parse_header(content_disposition)
- # We use ``or`` here because we don't want to use an "empty" value
- # from the filename param.
- filename = params.get('filename') or filename
- ext = splitext(filename)[1]
- if not ext:
- ext = mimetypes.guess_extension(content_type)
- if ext:
- filename += ext
- if not ext and link.url != resp.url:
- ext = os.path.splitext(resp.url)[1]
- if ext:
- filename += ext
- file_path = os.path.join(temp_dir, filename)
- with open(file_path, 'wb') as content_file:
- _download_url(resp, link, content_file, hashes)
- return file_path, content_type
-
-
-def _check_download_dir(link, download_dir, hashes):
- """ Check download_dir for previously downloaded file with correct hash
- If a correct file is found return its path else None
- """
- download_path = os.path.join(download_dir, link.filename)
- if os.path.exists(download_path):
- # If already downloaded, does its hash match?
- logger.info('File was already downloaded %s', download_path)
- if hashes:
- try:
- hashes.check_against_path(download_path)
- except HashMismatch:
- logger.warning(
- 'Previously-downloaded file %s has bad hash. '
- 'Re-downloading.',
- download_path
- )
- os.unlink(download_path)
- return None
- return download_path
- return None
OSDN Git Service
