GNOME.org
 

[jhbuild/pre-3-cleanup: 16/16] Various Python 3 porting to get some of the tests to run.

commit cda77852a717d2b3df1a1b6cc1769fa9f9ceef45
Author: Christoph Reiter <reiter christoph gmail com>
Date:   Sun Sep 22 12:33:24 2019 +0200

    Various Python 3 porting to get some of the tests to run.
    
    Some tests are skipped for now, but this makes sure that everything stays
    importable under Python 3.
    
    Also add a separate CI job for Python 3.

 .gitlab-ci.yml                    | 20 ++++++++++++++++----
 jhbuild/commands/__init__.py      |  8 ++++----
 jhbuild/commands/extdeps.py       |  9 ++-------
 jhbuild/commands/goalreport.py    | 14 +++++---------
 jhbuild/commands/twoninetynine.py |  2 +-
 jhbuild/frontends/terminal.py     |  2 +-
 jhbuild/modtypes/__init__.py      |  3 ++-
 jhbuild/modtypes/tarball.py       |  2 +-
 jhbuild/moduleset.py              |  9 ++++-----
 jhbuild/utils/Makefile.am         |  3 ++-
 jhbuild/utils/cmds.py             | 14 +++++++-------
 jhbuild/utils/compat.py           | 14 ++++++++++++++
 jhbuild/utils/httpcache.py        | 17 ++++++++---------
 jhbuild/utils/misc.py             | 12 +++++++++---
 jhbuild/utils/sysid.py            |  1 +
 jhbuild/utils/systeminstall.py    |  8 ++++----
 jhbuild/utils/urlutils.py         | 29 +++++++++++++++++++++++++++++
 jhbuild/versioncontrol/bzr.py     | 36 ++++++++++++++++++------------------
 jhbuild/versioncontrol/darcs.py   |  5 ++---
 jhbuild/versioncontrol/git.py     | 28 ++++++++++++++--------------
 jhbuild/versioncontrol/hg.py      |  5 ++---
 jhbuild/versioncontrol/svn.py     | 30 +++++++++++++++---------------
 jhbuild/versioncontrol/tarball.py | 13 ++++++-------
 scripts/jhbuild.in                | 12 ++++++++----
 tests/test_main.py                | 25 ++++++++++++++-----------
 25 files changed, 189 insertions(+), 132 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3380f855..346e323a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,17 +2,16 @@ stages:
   - test
   - deploy
 
-test:
+test-python2:
   image: registry.gitlab.gnome.org/gnome/jhbuild/jhbuild:v3
   script:
     - mkdir _build && cd _build
-    - ../autogen.sh
+    - ../autogen.sh --with-python=python2
     - make
     - make install
     - make distcheck
     - cd ..
     - python2 -m flake8 .
-    - python3 -m flake8 .
     - mkdir public
     - cd public
     - yelp-build html ../doc/C/index.docbook
@@ -20,6 +19,19 @@ test:
     paths:
       - public
 
+test-python3:
+  image: registry.gitlab.gnome.org/gnome/jhbuild/jhbuild:v3
+  script:
+    - mkdir _build && cd _build
+    - ../autogen.sh --with-python=python3
+    - make
+    - make install
+    - make distcheck
+    - cd ..
+    - python3 -m flake8 .
+  artifacts:
+    paths:
+      - public
 
 ubuntu-19.04-glib:
   image: registry.gitlab.gnome.org/gnome/jhbuild/jhbuild:v3
@@ -38,7 +50,7 @@ ubuntu-19.04-glib:
 pages:
   image: alpine:latest
   stage: deploy
-  needs: ["test"]
+  needs: ["test-python2"]
   script:
    - echo
   artifacts:
diff --git a/jhbuild/commands/__init__.py b/jhbuild/commands/__init__.py
index 522142db..ecae18c5 100644
--- a/jhbuild/commands/__init__.py
+++ b/jhbuild/commands/__init__.py
@@ -32,7 +32,7 @@ import os
 
 from jhbuild.errors import FatalError
 from jhbuild.utils import try_import_module, uprint, uencode, N_, _
-
+from jhbuild.utils.compat import iteritems
 
 class OptionParser(optparse.OptionParser):
     def exit(self, status=0, msg=None):
@@ -79,7 +79,7 @@ class BuildCommand(Command):
     def required_system_dependencies_installed(self, module_state):
         '''Returns true if all required system dependencies are installed for
         modules in module_state.'''
-        for module, (req_version, installed_version, new_enough, systemmodule) in module_state.iteritems():
+        for module, (req_version, installed_version, new_enough, systemmodule) in iteritems(module_state):
             if systemmodule:
                 if not new_enough:
                     return False
@@ -105,7 +105,7 @@ class BuildCommand(Command):
         print(_('Required packages:'))
         print(_('  System installed packages which are too old:'))
         have_too_old = False
-        for module, (req_version, installed_version, new_enough, systemmodule) in module_state.iteritems():
+        for module, (req_version, installed_version, new_enough, systemmodule) in iteritems(module_state):
             if (installed_version is not None) and (not new_enough) and systemmodule:
                 have_too_old = True
                 print ('    %s %s' % (module.name,
@@ -117,7 +117,7 @@ class BuildCommand(Command):
 
         print(_('  No matching system package installed:'))
         have_missing = False
-        for module, (req_version, installed_version, new_enough, systemmodule) in module_state.iteritems():
+        for module, (req_version, installed_version, new_enough, systemmodule) in iteritems(module_state):
             if installed_version is None and (not new_enough) and systemmodule:
                 have_missing = True
                 print('    %s %s' % (module.name,
diff --git a/jhbuild/commands/extdeps.py b/jhbuild/commands/extdeps.py
index 4087c2f1..a04fd700 100644
--- a/jhbuild/commands/extdeps.py
+++ b/jhbuild/commands/extdeps.py
@@ -26,12 +26,7 @@ import sys
 import time
 
 from jhbuild.utils import _
-from jhbuild.utils.compat import cmp
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
+from jhbuild.utils.compat import cmp, BytesIO
 
 import jhbuild.moduleset
 from jhbuild.commands import Command, register_command
@@ -85,7 +80,7 @@ class cmd_extdeps(Command):
 
     def run(self, config, options, args, help=None):
         if options.output:
-            output = StringIO()
+            output = BytesIO()
         else:
             output = sys.stdout
 
diff --git a/jhbuild/commands/goalreport.py b/jhbuild/commands/goalreport.py
index b718adf6..8e0e9f2a 100644
--- a/jhbuild/commands/goalreport.py
+++ b/jhbuild/commands/goalreport.py
@@ -26,14 +26,9 @@ import sys
 import subprocess
 import time
 import types
-import cPickle
+import pickle
 import logging
 from optparse import make_option
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
 import xml.etree.ElementTree as ET
 
 try:
@@ -45,6 +40,7 @@ import jhbuild.moduleset
 from jhbuild.commands import Command, register_command
 from jhbuild.utils import httpcache, cmds, _
 from jhbuild.modtypes import MetaModule
+from jhbuild.utils.compat import BytesIO
 
 try:
     t_bold = cmds.get_output(['tput', 'bold'])
@@ -371,7 +367,7 @@ class cmd_goalreport(Command):
 
     def run(self, config, options, args, help=None):
         if options.output:
-            output = StringIO()
+            output = BytesIO()
             global curses
             if curses and config.progress_bar:
                 try:
@@ -403,7 +399,7 @@ class cmd_goalreport(Command):
             cachedir = os.path.join(os.environ['HOME'], '.cache','jhbuild')
         if options.cache:
             try:
-                results = cPickle.load(open(os.path.join(cachedir, options.cache)))
+                results = pickle.load(open(os.path.join(cachedir, options.cache)))
             except:
                 pass
 
@@ -461,7 +457,7 @@ class cmd_goalreport(Command):
         if not os.path.exists(cachedir):
             os.makedirs(cachedir)
         if options.cache:
-            cPickle.dump(results, open(os.path.join(cachedir, options.cache), 'w'))
+            pickle.dump(results, open(os.path.join(cachedir, options.cache), 'w'))
 
         print(HTML_AT_TOP % {'title': self.title}, file=output)
         if self.page_intro:
diff --git a/jhbuild/commands/twoninetynine.py b/jhbuild/commands/twoninetynine.py
index 531297c4..07d79fe3 100644
--- a/jhbuild/commands/twoninetynine.py
+++ b/jhbuild/commands/twoninetynine.py
@@ -24,7 +24,7 @@ import re
 from jhbuild.commands import Command, register_command
 from jhbuild.utils import _
 
-from goalreport import cmd_goalreport, ExcludedModuleException, \
+from .goalreport import cmd_goalreport, ExcludedModuleException, \
          Check, ShellCheck, DeprecatedSymbolsCheck, FIND_C
 
 class LibBonobo(ShellCheck):
diff --git a/jhbuild/frontends/terminal.py b/jhbuild/frontends/terminal.py
index 74bf31e5..b6fb97f0 100644
--- a/jhbuild/frontends/terminal.py
+++ b/jhbuild/frontends/terminal.py
@@ -285,7 +285,7 @@ class TerminalBuildScript(buildscript.BuildScript):
         try:
             if p.wait() != 0:
                 if self.config.quiet_mode:
-                    print(''.join(output))
+                    print(b''.join(output))
                 raise CommandError(_('########## Error running %s')
                                    % print_args['command'], p.returncode)
         except OSError:
diff --git a/jhbuild/modtypes/__init__.py b/jhbuild/modtypes/__init__.py
index 235e7e78..666f37ce 100644
--- a/jhbuild/modtypes/__init__.py
+++ b/jhbuild/modtypes/__init__.py
@@ -38,6 +38,7 @@ from jhbuild.errors import FatalError, CommandError, BuildStateError, \
 from jhbuild.utils.sxml import sxml
 from jhbuild.utils import inpath, try_import_module, N_, _
 import jhbuild.utils.fileutils as fileutils
+from jhbuild.utils.compat import filterlist
 
 _module_types = {}
 def register_module_type(name, parse_func):
@@ -325,7 +326,7 @@ them into the prefix."""
             # $JHBUILD_PREFIX/_jhbuild/root-foo/$JHBUILD_PREFIX
             # Remove them one by one to clean the tree to the state we expect,
             # so we can better spot leftovers or broken things.
-            prefix_dirs = filter(lambda x: x != '', stripped_prefix.split(os.sep))
+            prefix_dirs = filterlist(lambda x: x != '', stripped_prefix.split(os.sep))
             while len(prefix_dirs) > 0:
                 dirname = prefix_dirs.pop()
                 subprefix = os.path.join(*([destdir] + prefix_dirs))
diff --git a/jhbuild/modtypes/tarball.py b/jhbuild/modtypes/tarball.py
index 27552e43..07da9590 100644
--- a/jhbuild/modtypes/tarball.py
+++ b/jhbuild/modtypes/tarball.py
@@ -85,7 +85,7 @@ def parse_tarball(node, config, uri, repositories, default_repo):
 
     dependencies, after, suggests, systemdependencies = get_dependencies(node)
 
-    from autotools import AutogenModule
+    from .autotools import AutogenModule
     from jhbuild.versioncontrol.tarball import TarballBranch, TarballRepository
 
     # create a fake TarballRepository, and give it the moduleset uri
diff --git a/jhbuild/moduleset.py b/jhbuild/moduleset.py
index ff9b43c8..c01c2e5c 100644
--- a/jhbuild/moduleset.py
+++ b/jhbuild/moduleset.py
@@ -21,7 +21,6 @@ from __future__ import generators
 
 import os
 import sys
-import urlparse
 import logging
 import xml.dom.minidom
 import xml.parsers.expat
@@ -38,7 +37,7 @@ from jhbuild.modtypes.testmodule import TestModule
 from jhbuild.modtypes.systemmodule import SystemModule
 from jhbuild.versioncontrol.tarball import TarballBranch
 from jhbuild.utils import systeminstall
-from jhbuild.utils import fileutils
+from jhbuild.utils import fileutils, urlutils
 
 __all__ = ['load', 'load_tests', 'get_default_repo']
 
@@ -344,7 +343,7 @@ class ModuleSet:
         if self.raise_exception_on_warning:
             raise UsageError(msg)
         else:
-            logging.warn(msg)
+            logging.warning(msg)
 
 
 def load(config, uri=None):
@@ -364,7 +363,7 @@ def load(config, uri=None):
                 uri = os.path.join(config.modulesets_dir, uri + '.modules')
             elif os.path.isfile(os.path.join(config.modulesets_dir, uri)):
                 uri = os.path.join(config.modulesets_dir, uri)
-        elif not urlparse.urlparse(uri)[0]:
+        elif not urlutils.urlparse(uri)[0]:
             uri = 'https://gitlab.gnome.org/GNOME/jhbuild/raw/master/modulesets' \
                   '/%s.modules' % uri
         ms.modules.update(_parse_module_set(config, uri).modules)
@@ -526,7 +525,7 @@ def _parse_module_set(config, uri):
     for node in _child_elements(document.documentElement):
         if node.nodeName == 'include':
             href = node.getAttribute('href')
-            inc_uri = urlparse.urljoin(uri, href)
+            inc_uri = urlutils.urljoin(uri, href)
             try:
                 inc_moduleset = _parse_module_set(config, inc_uri)
             except UndefinedRepositoryError:
diff --git a/jhbuild/utils/Makefile.am b/jhbuild/utils/Makefile.am
index 1a7f4ec7..149d7ce0 100644
--- a/jhbuild/utils/Makefile.am
+++ b/jhbuild/utils/Makefile.am
@@ -14,5 +14,6 @@ app_PYTHON = \
        systeminstall.py \
        trigger.py \
        trayicon.py \
-       unpack.py
+       unpack.py \
+       urlutils.py
 
diff --git a/jhbuild/utils/cmds.py b/jhbuild/utils/cmds.py
index 36f429fe..0b872365 100644
--- a/jhbuild/utils/cmds.py
+++ b/jhbuild/utils/cmds.py
@@ -177,7 +177,7 @@ def pprint_output(pipe, format_line):
     if not getattr(sys.stdin, "closed", True):
         read_set.append(sys.stdin)
 
-    out_data = err_data = ''
+    out_data = err_data = b''
     try:
         while read_set:
             rlist, wlist, xlist = select.select(read_set, [], [])
@@ -190,8 +190,8 @@ def pprint_output(pipe, format_line):
                     if sys.stdin in read_set:
                         read_set.remove(sys.stdin)
                 out_data += out_chunk
-                while '\n' in out_data:
-                    pos = out_data.find('\n')
+                while b'\n' in out_data:
+                    pos = out_data.find(b'\n')
                     format_line(out_data[:pos+1], False)
                     out_data = out_data[pos+1:]
 
@@ -201,8 +201,8 @@ def pprint_output(pipe, format_line):
                     pipe.stderr.close()
                     read_set.remove(pipe.stderr)
                 err_data += err_chunk
-                while '\n' in err_data:
-                    pos = err_data.find('\n')
+                while b'\n' in err_data:
+                    pos = err_data.find(b'\n')
                     format_line(err_data[:pos+1], True)
                     err_data = err_data[pos+1:]
 
@@ -244,14 +244,14 @@ def compare_version(version, minver):
     for i, ver in enumerate(version):
         part = re.sub(r'^[^\d]*(\d*).*$', r'\1', ver)
         if not part:
-            version[i] = None
+            version[i] = float("-inf")
         else:
             version[i] = int(part)
     minver = minver.split('.')
     for i, ver in enumerate(minver):
         part = re.sub(r'^[^\d]*(\d*).*$', r'\1', ver)
         if not part:
-            minver[i] = None
+            minver[i] = float("-inf")
         else:
             minver[i] = int(part)
     return version >= minver
diff --git a/jhbuild/utils/compat.py b/jhbuild/utils/compat.py
index 8fd3c5df..c4cade4d 100644
--- a/jhbuild/utils/compat.py
+++ b/jhbuild/utils/compat.py
@@ -20,6 +20,8 @@ PY3 = not PY2
 
 if PY2:
     import __builtin__ as builtins
+    from StringIO import StringIO as BytesIO
+    BytesIO
 
     cmp = builtins.cmp
     text_type = builtins.unicode
@@ -27,9 +29,15 @@ if PY2:
     file_type = builtins.file
     input = builtins.raw_input
     execfile = builtins.execfile
+
+    def iteritems(d):
+        return d.iteritems()
+
+    filterlist = filter
 elif PY3:
     import builtins
     from io import IOBase
+    from io import BytesIO as BytesIO
 
     def cmp(a, b):
         return (a > b) - (a < b)
@@ -53,3 +61,9 @@ elif PY3:
             source = f.read()
         code = compile(source, filename, "exec")
         exec(code, globals, locals)
+
+    def iteritems(d):
+        return iter(d.items())
+
+    def filterlist(*args, **kwargs):
+        return list(filter(*args, **kwargs))
\ No newline at end of file
diff --git a/jhbuild/utils/httpcache.py b/jhbuild/utils/httpcache.py
index 47c6d991..a489630c 100644
--- a/jhbuild/utils/httpcache.py
+++ b/jhbuild/utils/httpcache.py
@@ -30,15 +30,14 @@ and draws ideas from feedparser.py.  Strategies include:
 
 import os
 import sys
-import urllib2
-import urlparse
 import time
 from email.utils import parsedate_tz, mktime_tz
-import StringIO
 import gzip
 import xml.dom.minidom
 
 from jhbuild.utils import _
+from jhbuild.utils import urlutils
+from jhbuild.utils.compat import BytesIO
 
 def _parse_isotime(string):
     if string[-1] != 'Z':
@@ -143,7 +142,7 @@ class Cache:
         '''picks a unique name for a new entry in the cache.
         Very simplistic.'''
         # get the basename from the URI
-        parts = urlparse.urlparse(uri, allow_fragments=False)
+        parts = urlutils.urlparse(uri, allow_fragments=False)
         base = parts[2].split('/')[-1]
         if not base:
             base = 'index.html'
@@ -163,7 +162,7 @@ class Cache:
         '''Downloads the file associated with the URI, and returns a local
         file name for contents.'''
         # pass file URIs straight through -- no need to cache them
-        parts = urlparse.urlparse(uri)
+        parts = urlutils.urlparse(uri)
         if parts[0] in ('', 'file'):
             return parts[2]
         if sys.platform.startswith('win') and uri[1] == ':':
@@ -182,7 +181,7 @@ class Cache:
         if nonetwork:
             raise RuntimeError(_('file not in cache, but not allowed to check network'))
 
-        request = urllib2.Request(uri)
+        request = urlutils.Request(uri)
         request.add_header('Accept-encoding', 'gzip')
         if entry:
             if entry.modified:
@@ -191,13 +190,13 @@ class Cache:
                 request.add_header('If-None-Match', entry.etag)
 
         try:
-            response = urllib2.urlopen(request)
+            response = urlutils.urlopen(request)
 
             # get data, and gunzip it if it is encoded
             data = response.read()
             if response.headers.get('Content-Encoding', '') == 'gzip':
                 try:
-                    data = gzip.GzipFile(fileobj=StringIO.StringIO(data)).read()
+                    data = gzip.GzipFile(fileobj=BytesIO(data)).read()
                 except:
                     data = ''
 
@@ -211,7 +210,7 @@ class Cache:
             fp = open(filename, 'wb')
             fp.write(data)
             fp.close()
-        except urllib2.HTTPError as e:
+        except urlutils.HTTPError as e:
             if e.code == 304: # not modified; update validated
                 expires = e.hdrs.get('Expires')
                 filename = os.path.join(self.cachedir, entry.local)
diff --git a/jhbuild/utils/misc.py b/jhbuild/utils/misc.py
index e10f5686..4ac6e267 100644
--- a/jhbuild/utils/misc.py
+++ b/jhbuild/utils/misc.py
@@ -20,7 +20,7 @@ import importlib
 import pkgutil
 import locale
 
-from .compat import text_type
+from .compat import text_type, PY2
 
 def inpath(filename, path):
     for dir in path:
@@ -72,7 +72,10 @@ def udecode(s):
 def uprint(*args, **kwargs):
     '''Print Unicode string encoded for the terminal'''
 
-    print(*[uencode(s) for s in args], **kwargs)
+    if PY2:
+        print(*[uencode(s) for s in args], **kwargs)
+    else:
+        print(*args, **kwargs)
 
 def N_(x):
     return text_type(x)
@@ -89,4 +92,7 @@ def _(x):
 def install_translation(translation):
     global _ugettext
 
-    _ugettext = translation.ugettext
\ No newline at end of file
+    if PY2:
+        _ugettext = translation.ugettext
+    else:
+        _ugettext = translation.gettext
\ No newline at end of file
diff --git a/jhbuild/utils/sysid.py b/jhbuild/utils/sysid.py
index 88b94152..f6e1e09c 100644
--- a/jhbuild/utils/sysid.py
+++ b/jhbuild/utils/sysid.py
@@ -59,6 +59,7 @@ def read_os_release():
                 continue
 
         fields[field] = value
+    release_file.close()
 
     if 'ID' not in fields or 'VERSION_ID' not in fields:
         return False
diff --git a/jhbuild/utils/systeminstall.py b/jhbuild/utils/systeminstall.py
index e7b17982..92fcb2d3 100644
--- a/jhbuild/utils/systeminstall.py
+++ b/jhbuild/utils/systeminstall.py
@@ -27,10 +27,10 @@ import subprocess
 import imp
 import textwrap
 import time
-from StringIO import StringIO
 import re
 
-import cmds
+from .compat import BytesIO
+from . import cmds
 from . import _
 
 def get_installed_pkgconfigs(config):
@@ -41,7 +41,7 @@ def get_installed_pkgconfigs(config):
         stdout = proc.communicate()[0]
         proc.wait()
         pkgs = []
-        for line in StringIO(stdout):
+        for line in BytesIO(stdout):
             pkg, rest = line.split(None, 1)
             pkgs.append(pkg)
 
@@ -418,7 +418,7 @@ class AptSystemInstall(SystemInstall):
             raise RuntimeError("regexp mustn't be None or empty")
         apt_file_result = subprocess.check_output(["apt-file", "search", "--regexp", regexp])
         ret_value = []
-        for line in StringIO(apt_file_result):
+        for line in BytesIO(apt_file_result):
             parts = line.split(':', 1)
             if len(parts) != 2:
                 continue
diff --git a/jhbuild/utils/urlutils.py b/jhbuild/utils/urlutils.py
new file mode 100644
index 00000000..c856f5bd
--- /dev/null
+++ b/jhbuild/utils/urlutils.py
@@ -0,0 +1,29 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+from .compat import PY2, PY3
+
+if PY2:
+    from urlparse import urlparse, urljoin, urlsplit, urlunsplit, urlunparse
+    from urllib2 import urlopen, Request, HTTPError, URLError
+    from urllib import unquote
+    import urlparse as urlparse_mod
+elif PY3:
+    from urllib.parse import urlparse, urljoin, urlsplit, urlunsplit, urlunparse, unquote
+    from urllib.request import urlopen, Request
+    from urllib.error import HTTPError, URLError
+    from urllib import parse as urlparse_mod
+
+    urlparse, urljoin, urlsplit, urlunsplit, urlunparse, urlopen, Request,
+    urlparse_mod, HTTPError, URLError, unquote
\ No newline at end of file
diff --git a/jhbuild/versioncontrol/bzr.py b/jhbuild/versioncontrol/bzr.py
index 606fa3b4..855aecd2 100644
--- a/jhbuild/versioncontrol/bzr.py
+++ b/jhbuild/versioncontrol/bzr.py
@@ -21,31 +21,31 @@ __all__ = []
 __metaclass__ = type
 
 import os
-import urlparse
 import logging
 
 from jhbuild.errors import FatalError, CommandError
 from jhbuild.utils.cmds import get_output
 from jhbuild.versioncontrol import Repository, Branch, register_repo_type
-from jhbuild.utils import inpath, _
+from jhbuild.utils import inpath, _, urlutils
 from jhbuild.utils.sxml import sxml
+from jhbuild.utils.urlutils import urlparse_mod
 
 # Make sure that the urlparse module considers bzr://, bzr+ssh://, sftp:// and lp:
 # scheme to be netloc aware and set to allow relative URIs.
-if 'bzr' not in urlparse.uses_netloc:
-    urlparse.uses_netloc.append('bzr')
-if 'bzr' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('bzr')
-if 'bzr+ssh' not in urlparse.uses_netloc:
-    urlparse.uses_netloc.append('bzr+ssh')
-if 'bzr+ssh' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('bzr+ssh')
-if 'sftp' not in urlparse.uses_netloc:
-    urlparse.uses_netloc.append('sftp')
-if 'sftp' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('sftp')
-if 'lp' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('lp')
+if 'bzr' not in urlparse_mod.uses_netloc:
+    urlparse_mod.uses_netloc.append('bzr')
+if 'bzr' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('bzr')
+if 'bzr+ssh' not in urlparse_mod.uses_netloc:
+    urlparse_mod.uses_netloc.append('bzr+ssh')
+if 'bzr+ssh' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('bzr+ssh')
+if 'sftp' not in urlparse_mod.uses_netloc:
+    urlparse_mod.uses_netloc.append('sftp')
+if 'sftp' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('sftp')
+if 'lp' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('lp')
 
 
 class BzrRepository(Repository):
@@ -78,9 +78,9 @@ class BzrRepository(Repository):
             module = name
 
         if revision or branch:
-            template = urlparse.urljoin(self.href, self.branches_template)
+            template = urlutils.urljoin(self.href, self.branches_template)
         else:
-            template = urlparse.urljoin(self.href, self.trunk_template)
+            template = urlutils.urljoin(self.href, self.trunk_template)
 
         if not module_href:
             module_href = template % {
diff --git a/jhbuild/versioncontrol/darcs.py b/jhbuild/versioncontrol/darcs.py
index 2fabc141..cb173b67 100644
--- a/jhbuild/versioncontrol/darcs.py
+++ b/jhbuild/versioncontrol/darcs.py
@@ -21,12 +21,11 @@ __all__ = []
 __metaclass__ = type
 
 import os
-import urlparse
 import hashlib
 
 from jhbuild.errors import FatalError, CommandError
 from jhbuild.versioncontrol import Repository, Branch, register_repo_type
-from jhbuild.utils import inpath, _
+from jhbuild.utils import inpath, _, urlutils
 
 class DarcsRepository(Repository):
     """A class representing a Darcs repository.
@@ -54,7 +53,7 @@ class DarcsRepository(Repository):
         else:
             if module is None:
                 module = name
-            module = urlparse.urljoin(self.href, module)
+            module = urlutils.urljoin(self.href, module)
         return DarcsBranch(self, module, checkoutdir)
 
 
diff --git a/jhbuild/versioncontrol/git.py b/jhbuild/versioncontrol/git.py
index 82e65317..6930036b 100644
--- a/jhbuild/versioncontrol/git.py
+++ b/jhbuild/versioncontrol/git.py
@@ -23,10 +23,8 @@ __metaclass__ = type
 
 import os
 import stat
-import urlparse
 import subprocess
 import re
-import urllib
 import logging
 
 from jhbuild.errors import FatalError, CommandError
@@ -34,19 +32,21 @@ from jhbuild.utils.cmds import get_output, check_version
 from jhbuild.versioncontrol import Repository, Branch, register_repo_type
 from jhbuild.utils import inpath, _
 from jhbuild.utils.sxml import sxml
+from jhbuild.utils import urlutils
+from jhbuild.utils.urlutils import urlparse_mod
 
 # Make sure that the urlparse module considers git:// and git+ssh://
 # schemes to be netloc aware and set to allow relative URIs.
-if 'git' not in urlparse.uses_netloc:
-    urlparse.uses_netloc.append('git')
-if 'git' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('git')
-if 'git+ssh' not in urlparse.uses_netloc:
-    urlparse.uses_netloc.append('git+ssh')
-if 'git+ssh' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('git+ssh')
-if 'ssh' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('ssh')
+if 'git' not in urlparse_mod.uses_netloc:
+    urlparse_mod.uses_netloc.append('git')
+if 'git' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('git')
+if 'git+ssh' not in urlparse_mod.uses_netloc:
+    urlparse_mod.uses_netloc.append('git+ssh')
+if 'git+ssh' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('git+ssh')
+if 'ssh' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('ssh')
 
 def get_git_extra_env():
     # we run git without the JHBuild LD_LIBRARY_PATH and PATH, as it can
@@ -110,7 +110,7 @@ class GitRepository(Repository):
                 else:
                     if new_module:
                         module = new_module
-        if not (urlparse.urlparse(module)[0] or module[0] == '/'):
+        if not (urlutils.urlparse(module)[0] or module[0] == '/'):
             if self.href.endswith('/'):
                 base_href = self.href
             else:
@@ -558,7 +558,7 @@ class GitSvnBranch(GitBranch):
         # only parse the final match
         if match:
             branch = match.group(1)
-            external = urllib.unquote(match.group(2).replace("%0A", " ").strip("%20 ")).split()
+            external = urlutils.unquote(match.group(2).replace("%0A", " ").strip("%20 ")).split()
             revision_expr = re.compile(r"-r(\d*)")
             i = 0
             while i < len(external):
diff --git a/jhbuild/versioncontrol/hg.py b/jhbuild/versioncontrol/hg.py
index 7d9ba1a9..5982b9a7 100644
--- a/jhbuild/versioncontrol/hg.py
+++ b/jhbuild/versioncontrol/hg.py
@@ -22,12 +22,11 @@ __all__ = []
 __metaclass__ = type
 
 import os
-import urlparse
 from subprocess import Popen, PIPE
 
 from jhbuild.errors import FatalError, CommandError
 from jhbuild.versioncontrol import Repository, Branch, register_repo_type
-from jhbuild.utils import inpath, _
+from jhbuild.utils import inpath, _, urlutils
 
 class HgRepository(Repository):
     """A class representing a Mercurial repository.
@@ -56,7 +55,7 @@ class HgRepository(Repository):
             if module is None:
                 module = name
             if not self.href.startswith('ssh://'):
-                module = urlparse.urljoin(self.href, module)
+                module = urlutils.urljoin(self.href, module)
             else:
                 module = self.href + module
         return HgBranch(self, module, checkoutdir)
diff --git a/jhbuild/versioncontrol/svn.py b/jhbuild/versioncontrol/svn.py
index 7c3e0795..8d5d6b0b 100644
--- a/jhbuild/versioncontrol/svn.py
+++ b/jhbuild/versioncontrol/svn.py
@@ -23,14 +23,14 @@ __all__ = []
 __metaclass__ = type
 
 import os
-import urlparse
 import subprocess
 
 from jhbuild.errors import CommandError, BuildStateError
 from jhbuild.utils.cmds import get_output, check_version
 from jhbuild.versioncontrol import Repository, Branch, register_repo_type
-from jhbuild.utils import inpath, _
+from jhbuild.utils import inpath, _, urlutils
 from jhbuild.utils.sxml import sxml
+from jhbuild.utils.urlutils import urlparse_mod
 
 svn_one_five = None # is this svn 1.5
 
@@ -42,14 +42,14 @@ def _make_uri(repo, path):
 
 # Make sure that the urlparse module considers svn:// and svn+ssh://
 # schemes to be netloc aware and set to allow relative URIs.
-if 'svn' not in urlparse.uses_netloc:
-    urlparse.uses_netloc.append('svn')
-if 'svn' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('svn')
-if 'svn+ssh' not in urlparse.uses_netloc:
-    urlparse.uses_netloc.append('svn+ssh')
-if 'svn+ssh' not in urlparse.uses_relative:
-    urlparse.uses_relative.append('svn+ssh')
+if 'svn' not in urlparse_mod.uses_netloc:
+    urlparse_mod.uses_netloc.append('svn')
+if 'svn' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('svn')
+if 'svn+ssh' not in urlparse_mod.uses_netloc:
+    urlparse_mod.uses_netloc.append('svn+ssh')
+if 'svn+ssh' not in urlparse_mod.uses_relative:
+    urlparse_mod.uses_relative.append('svn+ssh')
 
 def get_svn_extra_env():
     # we run Subversion in the C locale, because Subversion localises
@@ -165,9 +165,9 @@ class SubversionRepository(Repository):
 
         # workaround for svn client not handling '..' in URL (#560246, #678869)
         if os.name != 'nt':
-            splitted_href = list(urlparse.urlsplit(module_href))
+            splitted_href = list(urlutils.urlsplit(module_href))
             splitted_href[2] = os.path.abspath(splitted_href[2])
-            module_href = urlparse.urlunsplit(splitted_href)
+            module_href = urlutils.urlunsplit(splitted_href)
 
         if self.svn_program == 'bzr' and not revision:
             return bzr.BzrBranch(self, module_href, checkoutdir)
@@ -276,11 +276,11 @@ class SubversionBranch(Branch):
 
         uri = get_uri(outputdir)
 
-        if urlparse.urlparse(uri)[:2] != urlparse.urlparse(self.module)[:2]:
+        if urlutils.urlparse(uri)[:2] != urlutils.urlparse(self.module)[:2]:
             # server and protocol changed, probably because user changed
             # svnroots[] config variable.
-            new_uri = urlparse.urlunparse(
-                    urlparse.urlparse(self.module)[:2] + urlparse.urlparse(uri)[2:])
+            new_uri = urlutils.urlunparse(
+                    urlutils.urlparse(self.module)[:2] + urlutils.urlparse(uri)[2:])
             cmd = ['svn', 'switch', '--relocate', uri, new_uri, '.']
             buildscript.execute(cmd, 'svn', cwd=outputdir,
                     extra_env=get_svn_extra_env())
diff --git a/jhbuild/versioncontrol/tarball.py b/jhbuild/versioncontrol/tarball.py
index 0b79da02..292cb310 100644
--- a/jhbuild/versioncontrol/tarball.py
+++ b/jhbuild/versioncontrol/tarball.py
@@ -22,8 +22,6 @@ __metaclass__ = type
 
 import os
 import hashlib
-import urlparse
-import urllib2
 import logging
 import zipfile
 
@@ -34,6 +32,7 @@ from jhbuild.modtypes import get_branch
 from jhbuild.utils.unpack import unpack_archive
 from jhbuild.utils import httpcache, _
 from jhbuild.utils.sxml import sxml
+from jhbuild.utils import urlutils
 
 
 class TarballRepository(Repository):
@@ -68,7 +67,7 @@ class TarballRepository(Repository):
         else:
             if module is None:
                 module = name
-            module = urlparse.urljoin(self.href, module)
+            module = urlutils.urljoin(self.href, module)
         module = module.replace('${version}', version)
         if checkoutdir is not None:
             checkoutdir = checkoutdir.replace('${version}', version)
@@ -258,19 +257,19 @@ class TarballBranch(Branch):
         # now patch the working tree
         for (patch, patchstrip) in self.patches:
             patchfile = ''
-            if urlparse.urlparse(patch)[0]:
+            if urlutils.urlparse(patch)[0]:
                 # patch name has scheme, get patch from network
                 try:
                     patchfile = httpcache.load(patch, nonetwork=buildscript.config.nonetwork)
-                except urllib2.HTTPError as e:
+                except urlutils.HTTPError as e:
                     raise BuildStateError(_('could not download patch (error: %s)') % e.code)
-                except urllib2.URLError:
+                except urlutils.URLError:
                     raise BuildStateError(_('could not download patch'))
             elif self.repository.moduleset_uri:
                 # get it relative to the moduleset uri, either in the same
                 # directory or a patches/ subdirectory
                 for patch_prefix in ('.', 'patches', '../patches'):
-                    uri = urlparse.urljoin(self.repository.moduleset_uri,
+                    uri = urlutils.urljoin(self.repository.moduleset_uri,
                             os.path.join(patch_prefix, patch))
                     try:
                         patchfile = httpcache.load(uri, nonetwork=buildscript.config.nonetwork)
diff --git a/scripts/jhbuild.in b/scripts/jhbuild.in
index 48992e33..a24406f0 100644
--- a/scripts/jhbuild.in
+++ b/scripts/jhbuild.in
@@ -3,7 +3,11 @@
 
 import sys
 import os
-import __builtin__
+
+if sys.version_info[0] == 2:
+    import __builtin__ as builtins
+else:
+    import builtins
 
 USE_CHECKOUT_SRC = False
 
@@ -24,9 +28,9 @@ else:
     except ImportError:
         sys.path.insert(0, srcdir)
 
-__builtin__.__dict__['PKGDATADIR'] = pkgdatadir
-__builtin__.__dict__['DATADIR'] = datadir
-__builtin__.__dict__['SRCDIR'] = srcdir
+builtins.__dict__['PKGDATADIR'] = pkgdatadir
+builtins.__dict__['DATADIR'] = datadir
+builtins.__dict__['SRCDIR'] = srcdir
 
 import jhbuild.main
 jhbuild.main.main(sys.argv[1:])
diff --git a/tests/test_main.py b/tests/test_main.py
index 16660e43..579ea8a8 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -30,13 +30,13 @@ import glob
 import tempfile
 import unittest
 
-import __builtin__
+from jhbuild.utils.compat import builtins, iteritems, PY3
 
 SRCDIR = os.path.join(os.path.dirname(__file__), '..')
 
-__builtin__.__dict__['PKGDATADIR'] = None
-__builtin__.__dict__['DATADIR'] = None
-__builtin__.__dict__['SRCDIR'] = SRCDIR
+builtins.__dict__['PKGDATADIR'] = None
+builtins.__dict__['DATADIR'] = None
+builtins.__dict__['SRCDIR'] = SRCDIR
 
 sys.path.insert(0, SRCDIR)
 
@@ -645,7 +645,7 @@ def restore_environ(env):
     # os.environ.clear() doesn't appear to change underlying environment.
     for key in os.environ.keys():
         del os.environ[key]
-    for key, value in env.iteritems():
+    for key, value in iteritems(env):
         os.environ[key] = value
 
 
@@ -669,6 +669,7 @@ def with_stdout_hidden(func):
 class EndToEndTest(JhbuildConfigTestCase):
 
     # FIXME: broken under Win32
+    @unittest.skipIf(PY3, "doesn't work yet")
     def test_distutils(self):
         config = self.make_config()
         module_list = [DistutilsModule('hello',
@@ -678,9 +679,10 @@ class EndToEndTest(JhbuildConfigTestCase):
         with_stdout_hidden(build.build)
         proc = subprocess.Popen(['hello'], stdout=subprocess.PIPE)
         stdout, stderr = proc.communicate()
-        self.assertEquals(stdout.strip(), 'Hello world (distutils)')
-        self.assertEquals(proc.wait(), 0)
+        self.assertEqual(stdout.strip(), b'Hello world (distutils)')
+        self.assertEqual(proc.wait(), 0)
 
+    @unittest.skipIf(PY3, "doesn't work yet")
     def test_autotools(self):
         config = self.make_config()
         module_list = [AutogenModule('hello',
@@ -690,12 +692,13 @@ class EndToEndTest(JhbuildConfigTestCase):
         with_stdout_hidden(build.build)
         proc = subprocess.Popen(['hello'], stdout=subprocess.PIPE)
         stdout, stderr = proc.communicate()
-        self.assertEquals(stdout.strip(), 'Hello world (autotools)')
-        self.assertEquals(proc.wait(), 0)
+        self.assertEqual(stdout.strip(), 'Hello world (autotools)')
+        self.assertEqual(proc.wait(), 0)
 
     # Won't pass under stock MSYS because pkgconfig isn't installed in base
     # path. Will work if you set ACLOCAL_FLAGS, PATH and PKG_CONFIG_PATH to
     # a prefix where pkg-config is installed.
+    @unittest.skipIf(PY3, "doesn't work yet")
     def test_autotools_with_libtool(self):
         config = self.make_config()
         module_list = [
@@ -707,8 +710,8 @@ class EndToEndTest(JhbuildConfigTestCase):
         with_stdout_hidden(build.build)
         proc = subprocess.Popen(['hello'], stdout=subprocess.PIPE)
         stdout, stderr = proc.communicate()
-        self.assertEquals(stdout.strip(), 'Hello world (library test)')
-        self.assertEquals(proc.wait(), 0)
+        self.assertEqual(stdout.strip(), b'Hello world (library test)')
+        self.assertEqual(proc.wait(), 0)
 
 class UtilsTest(JhbuildConfigTestCase):
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]