Update pip, & prep for customization

.gitignore

@@ -134,3 +134,6 @@ dmypy.json
 
 # pytype static type analyzer
 .pytype/
+
+#pytest test lib
+!tests/lib

pyproject.toml

@@ -34,6 +34,7 @@ norecursedirs = [
    "thirdparty",
    "tmp",
    "tests/data",
+   "tests/lib",
    ".eggs",
    "src/fetchcode/vcs/pip",
 ]
@@ -44,7 +45,18 @@ python_classes="Test"
 python_functions="test"
 
 addopts = [
-    "-rfExXw",
-    "--strict",
+    "-r fExXw",
+    "--strict-markers",
     "--doctest-modules"
 ]
+
+markers = [
+    "network: tests that need network",
+    "no_auto_tempdir_manager",
+    "unit: unit tests",
+    "integration: integration tests",
+    "bzr: VCS: Bazaar",
+    "svn: VCS: Subversion",
+    "mercurial: VCS: Mercurial",
+    "git: VCS: git",
+]

setup.cfg

@@ -26,6 +26,7 @@ install_requires =
     packageurl-python
     requests
     six
+    tenacity
 setup_requires = setuptools_scm[toml] >= 4
 
 [options.packages.find]
@@ -35,7 +36,9 @@ where=src
 testing =
     # upstream
     pytest
+    pytest-rerunfailures
     pytest-xdist
+    scripttest
 docs =
     Sphinx>=3.3.1
     sphinx-rtd-theme>=0.5.0

src/fetchcode/__init__.py

@@ -41,7 +41,7 @@ def __init__(self, location, content_type, size, url):
 def fetch_http(url, location):
     """
     Return a `Response` object built from fetching the content at a HTTP/HTTPS based `url` URL string
-    saving the content in a file at `location`   
+    saving the content in a file at `location`
     """
     r = requests.get(url)
     with open(location, 'wb') as f:
@@ -59,7 +59,7 @@ def fetch_http(url, location):
 def fetch_ftp(url, location):
     """
     Return a `Response` object built from fetching the content at a FTP based `url` URL string
-    saving the content in a file at `location`  
+    saving the content in a file at `location`
     """
     url_parts = urlparse(url)
 

src/fetchcode/vcs/__init__.py

@@ -29,9 +29,9 @@
 class VCSResponse:
     """
     Represent the response from fetching a VCS URL with:
-- `dest_dir`: destination of directory
-- `vcs_type`: VCS Type of URL (git,bzr,hg,svn)
-- `domain` : Source of git VCS (GitHub, Gitlab, Bitbucket)
+    - `dest_dir`: destination of directory
+    - `vcs_type`: VCS Type of URL (git,bzr,hg,svn)
+    - `domain` : Source of git VCS (GitHub, Gitlab, Bitbucket)
     """
 
     def __init__(self, dest_dir, vcs_type, domain):
@@ -43,7 +43,7 @@ def __init__(self, dest_dir, vcs_type, domain):
 def fetch_via_vcs(url):
     """
     Take `url` as input and store the content of it at location specified at `location` string
-    Return a VCSResponse object 
+    Return a VCSResponse object
     """
     parsed_url = urlparse(url)
     scheme = parsed_url.scheme