[library-web] support merging pages from different mallard documents

[Frederic Peters <fpeters src gnome org>]

commit d80096b0ec7ab3350ea3c68f2a1bc4c63ce68dc6
Author: Frédéric Péters <fpeters 0d be>
Date:   Wed Jan 29 15:53:43 2014 +0000

    support merging pages from different mallard documents
    
    gnome-getting-started-docs extends gnome-user-docs, it adds several pages,
    pictures and videos to gnome-help, and that's the combined document that
    should be published.

 src/lgo.py              |   27 +++++++++++--
 src/modtypes/base.py    |    9 +++-
 src/modtypes/mallard.py |   95 ++++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 118 insertions(+), 13 deletions(-)
---
diff --git a/src/lgo.py b/src/lgo.py
index 0aa3ffa..3814b53 100755
--- a/src/lgo.py
+++ b/src/lgo.py
@@ -141,6 +141,7 @@ class Lgo(App):
 
         if self.options.rebuild_module:
             self.rebuild_all = True
+            doc_modules = []
             for module in self.rebuild_modules:
                 if urlparse.urlparse(module).scheme:
                     filename = self.download(module)
@@ -149,8 +150,9 @@ class Lgo(App):
                         sys.exit(1)
                 else:
                     filename = module
-                for doc_module in self.extract_modules(filename):
-                    doc_module.process()
+                doc_modules.extend(self.extract_modules(filename))
+            for doc_module in doc_modules:
+                doc_module.process()
             sys.exit(0)
 
         if self.options.rebuild_remote:
@@ -606,9 +608,6 @@ class Lgo(App):
             tarball_name = tarball_name.split('__')[-1]
         tarname_without_minor = re.match(r'(.*)\.\d+', tarball_name).groups()[0]
 
-        if tarname_without_minor.startswith('gnome-getting-started-docs'):
-            return []
-
         # Step 0: remove old occurences of module extracts
         for basedir, dirnames, filenames in os.walk(os.path.join(app.config.private_dir, 'extracts')):
             for extract_filename in filenames:
@@ -709,6 +708,23 @@ class Lgo(App):
                 else:
                     continue
 
+                if doc.modulename == 'gnome-help' and tarname_without_minor.startswith(
+                        'gnome-getting-started-docs'):
+                    # special handling
+                    try:
+                        gnome_help_doc = [x for x in app.documents if x.module == 'gnome-help'][0]
+                    except IndexError:
+                        pass
+                    else:
+                        gnome_help_doc.version_subs.get(doc.one_dot_version).extend_mallard(doc)
+                        doc.filename = filename
+                        doc.mtime_tarball = mtime
+                        if extraction_happened:
+                            doc.extract(force=True)
+                        else:
+                            extraction_happened = doc.extract()
+                    continue
+
                 if '$(' in doc.modulename:
                     logging.debug('skipping because it has $( in its modulename')
                     continue
@@ -739,6 +755,7 @@ class Lgo(App):
                 doc.path = self.get_module_web_path(doc)
                 if self.config.channels is None or doc.channel in self.config.channels:
                     doc_modules.append(doc)
+                    doc.get_libgo_document([])
                 else:
                     logging.debug('ignoring %s, not in an appropriate channel' % doc.modulename)
 
diff --git a/src/modtypes/base.py b/src/modtypes/base.py
index ca337eb..34bae01 100644
--- a/src/modtypes/base.py
+++ b/src/modtypes/base.py
@@ -40,6 +40,7 @@ class DocModule(object):
     nightly = False
     extra_devel_releases = None
     git_version = False
+    mallard_merge = None
 
     def create_from_tar(cls, tar, tarinfo, makefile_am, nightly = False):
         self = cls()
@@ -134,11 +135,12 @@ class DocModule(object):
             doc.tarballname = self.tarballname
             doc._last_version = self.version
             doc.versions = [self.one_dot_version]
+            doc.version_subs = {}
             app.documents.append(doc)
         else:
-            if doc._last_version == self.version and not self.git_version:
-                # file was already processed in a previous moduleset
-                return None
+            #if doc._last_version == self.version and not self.git_version:
+            #    # file was already processed in a previous moduleset
+            #    return None
 
             if not self.nightly:
                 # a more recent version may already have been installed; probably
@@ -159,6 +161,7 @@ class DocModule(object):
                     doc.languages.append(lang)
 
         doc.version_mapping[self.one_dot_version] = self.version
+        doc.version_subs[self.one_dot_version] = self
 
         # only keep authorised languages
         if app.config.languages:
diff --git a/src/modtypes/mallard.py b/src/modtypes/mallard.py
index 5c34254..72c4c1c 100644
--- a/src/modtypes/mallard.py
+++ b/src/modtypes/mallard.py
@@ -192,6 +192,7 @@ class MallardModule(DocModule):
     category = None
 
     related_xsl_files = ['mal2html.xsl', 'heading.xsl']
+    extended_by_documents = None
 
     def setup_channel(self):
         # get category from omf file
@@ -330,17 +331,39 @@ class MallardModule(DocModule):
                 logging.debug('using already generated doc (lang: %s)' % lang)
                 continue
 
+            potential_xml_files = []
             mallard_cache = MallardCache()
             for doc_page in doc_pages:
-                xml_file = os.path.join(lang_dirname, doc_page)
                 if doc_page == '$(NULL)':
                     doc_pages.remove(doc_page)
                     continue
+                xml_file = os.path.join(lang_dirname, doc_page)
                 if not os.path.exists(xml_file):
                     if lang in doc.languages:
                         doc.languages.remove(lang)
                     logging.warn('failed to find %s for lang %s' % (xml_file, lang))
                     continue
+                potential_xml_files.append(xml_file)
+
+            if self.extended_by_documents:
+                for extra_document in self.extended_by_documents:
+                    extra_doc_pages = re.findall(r'(?:DOC_PAGES|HELP_FILES)\s+=\s+(.*)',
+                            extra_document.makefile_am)[0].split()
+                    if lang == 'en' and not os.path.exists(
+                            os.path.join(ext_dirname, extra_document.dirname, 'en')):
+                        extra_lang_dirname = os.path.join(ext_dirname, extra_document.dirname, 'C')
+                    else:
+                        extra_lang_dirname = os.path.join(ext_dirname, extra_document.dirname, lang)
+                    if not os.path.exists(os.path.join(extra_lang_dirname, extra_doc_pages[0])):
+                        extra_document.generate_translations(
+                                os.path.join(ext_dirname, extra_document.dirname), lang)
+                    for doc_page in extra_doc_pages:
+                        xml_file = os.path.join(extra_lang_dirname, doc_page)
+                        if os.path.exists(xml_file):
+                            potential_xml_files.append(xml_file)
+
+            xml_files = []
+            for xml_file in potential_xml_files:
                 try:
                     page = mallard_cache.add_page(xml_file)
                 except xml.parsers.expat.ExpatError:
@@ -349,10 +372,12 @@ class MallardModule(DocModule):
                     continue
                 except NotAMallardPageException:
                     logging.warn('failed processing %s, not a mallard page' % xml_file)
-                    doc_pages.remove(doc_page)
                     continue
+                xml_files.append(xml_file)
+
             if not lang in doc.languages:
                 continue
+
             temporary = tempfile.NamedTemporaryFile()
             mallard_cache.dump(temporary.name)
 
@@ -385,9 +410,36 @@ class MallardModule(DocModule):
                 open(dst, 'w').write(open(src, 'r').read())
                 copied_files.append(dst)
 
-            for doc_page in doc_pages:
-                xml_file = os.path.join(lang_dirname, doc_page)
-                filename = os.path.splitext(doc_page)[0] + '.html.' + lang
+            if self.extended_by_documents:
+                for extra_document in self.extended_by_documents:
+                    extra_doc_figures = re.findall(r'(?:DOC_FIGURES|HELP_FIGURES|HELP_MEDIA)\s+=\s+(.*)',
+                            extra_document.makefile_am)[0].split()
+                    self.expand_doc_figures(extra_doc_figures)
+
+                    if lang == 'en' and not os.path.exists(
+                            os.path.join(ext_dirname, extra_document.dirname, 'en')):
+                        extra_lang_dirname = os.path.join(ext_dirname, extra_document.dirname, 'C')
+                    else:
+                        extra_lang_dirname = os.path.join(ext_dirname, extra_document.dirname, lang)
+
+                    for doc_figure in extra_doc_figures:
+                        src = os.path.join(extra_lang_dirname, doc_figure)
+                        if os.path.exists(src):
+                            # file already exists, keep it
+                            continue
+                        # fallback to file from C locale.
+                        src = os.path.join(ext_dirname, extra_document.dirname, 'C', doc_figure)
+                        if not os.path.exists(src):
+                            # fallback file doesn't even exist, ignore
+                            continue
+
+                        dst = os.path.join(lang_dirname, doc_figure)
+                        if not os.path.exists(os.path.split(dst)[0]):
+                            os.makedirs(os.path.split(dst)[0])
+                        open(dst, 'w').write(open(src, 'r').read())
+                        copied_files.append(dst)
+
+            for xml_file in xml_files:
 
                 # format docbook into html files
                 cmd = ['xsltproc',
@@ -443,5 +495,38 @@ class MallardModule(DocModule):
                         os.makedirs(os.path.split(dst)[0])
                     open(dst, 'w').write(open(src, 'r').read())
 
+                if self.extended_by_documents:
+                    for extra_document in self.extended_by_documents:
+                        extra_doc_figures = re.findall(r'(?:DOC_FIGURES|HELP_FIGURES|HELP_MEDIA)\s+=\s+(.*)',
+                                extra_document.makefile_am)[0].split()
+                        extra_doc_figures.extend(re.findall(r'(?:HELP_FILES)\s+=\s+(.*)',
+                                extra_document.makefile_am)[0].split())
+                        self.expand_doc_figures(extra_doc_figures)
+                        extra_doc_figures = [x for x in extra_doc_figures if not x.endswith('.page')]
+
+                        if lang == 'en' and not os.path.exists(
+                                os.path.join(ext_dirname, extra_document.dirname, 'en')):
+                            extra_lang_dirname = os.path.join(ext_dirname, extra_document.dirname, 'C')
+                        else:
+                            extra_lang_dirname = os.path.join(ext_dirname, extra_document.dirname, lang)
+
+                        for doc_figure in extra_doc_figures:
+                            src = os.path.join(extra_lang_dirname, doc_figure)
+                            if not os.path.exists(src):
+                                # fallback to file from C locale.
+                                src = os.path.join(ext_dirname, extra_document.dirname, 'C', doc_figure)
+                                if not os.path.exists(src):
+                                    # fallback file doesn't even exist, ignore
+                                    continue
+
+                            dst = os.path.join(web_output_dir, doc_figure + '.%s' % lang)
+                            if not os.path.exists(os.path.split(dst)[0]):
+                                os.makedirs(os.path.split(dst)[0])
+
+
         self.install_version_symlinks(doc)
 
+    def extend_mallard(self, doc):
+        if not self.extended_by_documents:
+            self.extended_by_documents = []
+        self.extended_by_documents.append(doc)