Upgrade dependencies (#839)

- Update and pin dependencies to specific versions that support Python 3.7-3.11
- Replace deprecated werkzeug.pop_path_info with wsgiref.shift_path_info
- Use the latest httpbin from psf/httpbin
- Remove unused flask test dependency
- Drop Python 2 and Python <3.7 support
- Ensure greenlet 2 is used for now, as psf/httpbin doesn't yet work with greenlet 3

---------

Co-authored-by: Tessa Walsh <[email protected]>

.github/workflows/ci.yaml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       max-parallel: 3
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
 
     steps:
       - name: checkout

pywb/apps/frontendapp.py

@@ -1,7 +1,7 @@
 from gevent.monkey import patch_all; patch_all()
 
 from werkzeug.routing import Map, Rule, RequestRedirect, Submount
-from werkzeug.wsgi import pop_path_info
+from wsgiref.util import shift_path_info
 from six.moves.urllib.parse import urljoin, parse_qsl
 from six import iteritems
 from warcio.utils import to_native_str
@@ -558,9 +558,9 @@ def setup_paths(self, environ, coll, record=False):
             return
 
         if coll != '$root':
-            pop_path_info(environ)
+            shift_path_info(environ)
             if record:
-                pop_path_info(environ)
+                shift_path_info(environ)
 
         paths = [self.warcserver.root_dir]
 
@@ -669,7 +669,7 @@ def handle_request(self, environ, start_response):
 
             lang = args.pop('lang', '')
             if lang:
-                pop_path_info(environ)
+                shift_path_info(environ)
 
             if lang:
                 environ['pywb_lang'] = lang

pywb/apps/rewriterapp.py

@@ -64,7 +64,7 @@ def __init__(self, framed_replay=False, jinja_env=None, config=None, paths=None)
 
         if not jinja_env:
             jinja_env = JinjaEnv(globals={'static_path': 'static'},
-                                 extensions=['jinja2.ext.i18n', 'jinja2.ext.with_'])
+                                 extensions=['jinja2.ext.i18n'])
             jinja_env.jinja_env.install_null_translations()
 
         self.jinja_env = jinja_env

pywb/rewrite/templateview.py

@@ -5,7 +5,7 @@
 
 from six.moves.urllib.parse import urlsplit, quote
 
-from jinja2 import Environment, TemplateNotFound, contextfunction, select_autoescape
+from jinja2 import Environment, TemplateNotFound, pass_context, select_autoescape
 from jinja2 import FileSystemLoader, PackageLoader, ChoiceLoader
 
 from webassets.ext.jinja2 import AssetsExtension
@@ -139,7 +139,7 @@ def get_translate(context):
             return loc_map.get(loc)
 
         def override_func(jinja_env, name):
-            @contextfunction
+            @pass_context
             def get_override(context, text):
                 translate = get_translate(context)
                 if not translate:
@@ -158,7 +158,7 @@ def get_override(context, text):
 
         # Special _Q() function to return %-encoded text, necessary for use
         # with text in banner
-        @contextfunction
+        @pass_context
         def quote_gettext(context, text):
             translate = get_translate(context)
             if not translate:
@@ -171,7 +171,7 @@ def quote_gettext(context, text):
         self.jinja_env.globals['_Q'] = quote_gettext
         self.jinja_env.globals['default_locale'] = default_locale
 
-        @contextfunction
+        @pass_context
         def switch_locale(context, locale):
             environ = context.get('env')
             curr_loc = environ.get('pywb_lang', '')
@@ -188,7 +188,7 @@ def switch_locale(context, locale):
 
             return app_prefix + '/' + locale + request_uri
 
-        @contextfunction
+        @pass_context
         def get_locale_prefixes(context):
             environ = context.get('env')
             locale_prefixes = {}

pywb/warcserver/test/test_inputreq.py

@@ -39,7 +39,7 @@ def __call__(self, environ, start_response):
 
 #=============================================================================
 class TestInputReq(object):
-    def setup(self):
+    def setup_method(self):
         self.app = InputReqApp()
         self.testapp = webtest.TestApp(self.app)
 

pywb/warcserver/test/test_upstream.py

@@ -18,7 +18,7 @@
 
 
 class TestUpstream(LiveServerTests, HttpBinLiveTests, BaseTestClass):
-    def setup(self):
+    def setup_method(self):
         app = BaseWarcServer()
 
         base_url = 'http://localhost:{0}'.format(self.server.port)

requirements.txt

@@ -1,19 +1,20 @@
 six
 warcio>=1.7.1
 requests
-redis<3.0
-jinja2<3.0.0
+redis
+jinja2>=3.1.2
 surt>=0.3.1
 brotlipy
 pyyaml
-werkzeug
+werkzeug==2.2.3
 webencodings
-gevent==21.12.0
+gevent==22.10.2
+greenlet>=2.0.2,<3.0
 webassets==2.0
 portalocker
 wsgiprox>=1.5.1
 fakeredis<1.0
 tldextract
 python-dateutil
-markupsafe<2.1.0
+markupsafe>=2.1.1
 ua_parser

setup.py

@@ -113,6 +113,7 @@ def get_package_data():
             "translate_toolkit"
         ],
     },
+    python_requires='>=3.7,<3.12',
     tests_require=load_requirements("test_requirements.txt"),
     cmdclass={'test': PyTest},
     test_suite='',
@@ -131,16 +132,12 @@ def get_package_data():
         'Environment :: Web Environment',
         'License :: OSI Approved :: GNU General Public License (GPL)',
         'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Topic :: Internet :: Proxy Servers',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Internet :: WWW/HTTP :: WSGI',

test_requirements.txt

@@ -3,7 +3,6 @@ WebTest
 pytest-cov
 mock
 urllib3
-httpbin==0.5.0
-flask<2.0
 ujson
 lxml
+httpbin>=0.10.2

tests/test_force_https.py

@@ -56,6 +56,6 @@ def test_force_https_root_replay_1(self, fmod):
         resp = self.get('/20140128051539{0}/http://www.iana.org/domains/example', fmod,
                         headers={'X-Forwarded-Proto': 'https'})
 
-        assert resp.headers['Location'] == 'https://localhost:80/20140128051539{0}/http://www.iana.org/domains/reserved'.format(fmod)
+        assert resp.headers['Location'] == 'https://localhost:80/20140128051539{0}/http://www.iana.org/help/example-domains'.format(fmod)
 
 

tests/test_live_rewriter.py

@@ -91,25 +91,28 @@ def test_live_head(self, fmod_sl):
         resp = self.head('/live/{0}httpbin.org/get?foo=bar', fmod_sl)
         assert resp.status_int == 200
 
-    @pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
-    def test_live_bad_content_length(self, fmod_sl):
-        resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl, status=200)
-        assert resp.headers['Content-Length'] == '149'
-
-        resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl, status=200)
-        assert resp.headers['Content-Length'] == '90'
-
-    @pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
-    def test_live_bad_content_length_with_range(self, fmod_sl):
-        resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl,
-                        headers={'Range': 'bytes=0-'}, status=206)
-        assert resp.headers['Content-Length'] == '149'
-        assert resp.headers['Content-Range'] == 'bytes 0-148/149'
-
-        resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl,
-                        headers={'Range': 'bytes=0-'}, status=206)
-        assert resp.headers['Content-Length'] == '90'
-        assert resp.headers['Content-Range'] == 'bytes 0-89/90'
+    # Following tests are temporarily commented out because latest version of PSF httpbin
+    # now returns 400 if content-length header isn't parsable as an int
+
+    # @pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
+    # def test_live_bad_content_length(self, fmod_sl):
+    #     resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl, status=200)
+    #     assert resp.headers['Content-Length'] == '149'
+
+    #     resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl, status=200)
+    #     assert resp.headers['Content-Length'] == '90'
+
+    # @pytest.mark.skipif(sys.version_info < (3,0), reason='does not respond in 2.7')
+    # def test_live_bad_content_length_with_range(self, fmod_sl):
+    #     resp = self.get('/live/{0}httpbin.org/response-headers?content-length=149,149', fmod_sl,
+    #                     headers={'Range': 'bytes=0-'}, status=206)
+    #     assert resp.headers['Content-Length'] == '149'
+    #     assert resp.headers['Content-Range'] == 'bytes 0-148/149'
+
+    #     resp = self.get('/live/{0}httpbin.org/response-headers?Content-Length=xyz', fmod_sl,
+    #                     headers={'Range': 'bytes=0-'}, status=206)
+    #     assert resp.headers['Content-Length'] == '90'
+    #     assert resp.headers['Content-Range'] == 'bytes 0-89/90'
 
     def test_custom_unicode_header(self, fmod_sl):
         value = u'⛄'

tox.ini

@@ -4,15 +4,15 @@ testpaths =
     tests
 
 [tox]
-envlist = py36, py37, py38, py39, py310
+envlist = py37, py38, py39, py310, py311
 
 [gh-actions]
 python =
-    3.6: py36
     3.7: py37
     3.8: py38
     3.9: py39
     3.10: py310
+    3.11: py311
 
 [testenv]
 setenv = PYWB_NO_VERIFY_SSL = 1
@@ -22,6 +22,6 @@ deps =
     -rrequirements.txt
     -rextra_requirements.txt
 commands =
-    py.test --cov-config .coveragerc --cov pywb -v --doctest-modules ./pywb/ tests/
+    pytest --cov-config .coveragerc --cov pywb -v --doctest-modules ./pywb/ tests/