GNOME.org
 

ooo-build r13567 - in trunk: . bin

Author: jannieuw
Date: Thu Aug 14 12:50:24 2008
New Revision: 13567
URL: http://svn.gnome.org/viewvc/ooo-build?rev=13567&view=rev

Log:
2008-08-14  Jan Nieuwenhuizen  <janneke gnu org>

	* bin/src-pack2:  Split-out modules data to
	* bin/modules2.txt: New file with modules data.

	* bin/gob: New option: --split adds support for git-export in
	src-pack2 format, using bin/modules2.txt.


Added:
   trunk/bin/modules2.txt
Modified:
   trunk/ChangeLog
   trunk/bin/gob
   trunk/bin/src-pack2

Modified: trunk/bin/gob
==============================================================================
--- trunk/bin/gob	(original)
+++ trunk/bin/gob	Thu Aug 14 12:50:24 2008
@@ -10,10 +10,18 @@
 import os
 import re
 import sys
+import shutil
 
 class SystemFailed (Exception):
     pass
 
+def log (s, threshold=1):
+    if options.verbose > threshold:
+        print >> sys.stderr, s
+    
+def info (s):
+    log (s, threshold=0)
+
 def exception_string (exception=Exception ('no message')):
     import traceback
     return traceback.format_exc (None)
@@ -21,35 +29,52 @@
 def filter_out (predicate, lst):
     return filter (lambda x: not predicate (x), lst)
     
+def makedirs (dir):
+    log ('mkdir %(dir)s' % locals ())
+    os.makedirs (dir)
+    
+def symlink (src, dest):
+    log ('ln -s %(src)s %(dest)s' % locals ())
+    os.symlink (src, dest)
+
+def rename (src, dest):
+    log ('mv %(src)s %(dest)s' % locals ())
+    os.rename (src, dest)
+    
+def rmdir (dir):
+    log ('rmdir %(dir)s' % locals ())
+    os.rmdir (dir)
+    
 def system (command, raise_on_error=True):
     if options.verbose > 1:
-        print >> sys.stderr, 'executing: %(command)s' % locals ()
-    if options.verbose < 2:
+        log ('executing: %(command)s' % locals ())
+    if options.verbose < 3:
         command = '(%(command)s) > gob.log 2>&1' % locals ()
     status = os.system (command)
     if status and raise_on_error:
-        print >> sys.stderr, command
-        print >> sys.stderr, file ('gob.log').read ()
+        info (command)
+        info (file ('gob.log').read ())
         raise SystemFailed ('Command failed: %(command)s' % locals ())
     return status
 
 def read_pipe (command, raise_on_error=True):
-    if options.verbose > 1:
-        print >> sys.stderr, 'executing: %(command)s' % locals ()
-    if options.verbose < 2:
+    log ('executing: %(command)s' % locals ())
+    if options.verbose < 3:
         command = '(%(command)s) 2> gob.log' % locals ()
     pipe = os.popen (command)
     output = pipe.read ()
-    if options.verbose > 2:
-        print >> sys.stderr, 'pipe-output:\n%(output)s)s' % locals ()
+    log ('pipe-output:\n%(output)s)s' % locals (), threshold=2)
     if pipe.close () and raise_on_error:
-        print >> sys.stderr, command
-        print >> sys.stderr, file ('gob.log').read ()
+        info (command)
+        info (file ('gob.log').read ())
         raise SystemFailed ('Pipe failed: %(command)s' % locals ())
     return output
 
-def list_dirs (dir):
-    return filter (lambda x: os.path.isdir (os.path.join (dir, x)), os.listdir (dir))
+def list_dirs (dir, allow_link=True):
+    return filter (lambda x: (os.path.isdir (os.path.join (dir, x))
+                              and (allow_link
+                                   or not os.path.islink (os.path.join (dir, x)))),
+                   os.listdir (dir))
 
 def find_file (root, path, name):
     for dir in path:
@@ -58,9 +83,70 @@
             return file_name
     return None
 
-def apply_patch (dir, patch):
+def _apply_patch (dir, patch):
     system ('patch -l -p0 -d %(dir)s < %(patch)s' % locals ())
 
+def apply_patch (dir, patch):
+    if not options.split:
+        return _apply_patch (dir, patch)
+    apply_dir = os.path.join (dir, options.flat_apply_dir)
+    _apply_patch (apply_dir, patch)
+    for module in list_dirs (apply_dir, allow_link=False):
+        rename (os.path.join (apply_dir, module), os.path.join (dir, module))
+
+def get_srcpack2_dict ():
+    def modules_line (x):
+        e = x.split ("=")
+        return e[0], e[1].split (',')
+    return dict (map (modules_line, file (src_dir + 'bin/modules2.txt').readlines ()))
+
+def for_each (func, lst):
+    for i in lst:
+        func (i)
+
+def move (src):
+    src2 = get_srcpack2_dict ()
+    def move_pack (pack):
+        pack_dir = os.path.join (src, pack)
+        def move_module (module):
+            module_dir = src + '/' + module
+            pack_module_dir = pack_dir + '-/' + module
+            if os.path.exists (module_dir):
+                rename (module_dir, pack_module_dir)
+        makedirs (pack_dir + '-')
+        for_each (move_module, src2[pack])
+        rename (pack_dir + '-', pack_dir)
+    for_each (move_pack, src2.keys ())
+
+def move_back (src):
+    src2 = get_srcpack2_dict ()
+    def move_pack (pack):
+        pack_dir = os.path.join (src, pack)
+        def move_module (module):
+            print 'renaming:', module
+            module_dir = src + '/' + module
+            pack_module_dir = pack_dir + '-/' + module
+            if os.path.exists (pack_module_dir):
+                rename (pack_module_dir, module_dir)
+            else:
+                print 'no such dir:', pack_module_dir
+        rename (pack_dir, pack_dir + '-')
+        for_each (move_module, src2[pack])
+        rmdir (pack_dir + '-')
+    for_each (move_pack, src2.keys ())
+
+def setup_flat_apply_dir (src):
+    src2 = get_srcpack2_dict ()
+    apply_dir = os.path.join (src, options.flat_apply_dir)
+    shutil.rmtree (apply_dir, ignore_errors=True)
+    makedirs (apply_dir)
+    missing = ['mdbtools', 'libwpg', 'libwps', 'xalan']
+    for pack in src2.keys ():
+        for module in src2[pack]:
+            symlink (os.path.join ('..', '..', pack, module), os.path.join (apply_dir, module))
+    for pack in missing:
+        symlink (os.path.join ('..', '..', pack), os.path.join (apply_dir, pack))
+
 def patch_get_branch (patch):
     patch_file = patch.file_name
     if not patch_file:
@@ -82,6 +168,8 @@
     module_re = None
     if not module_re:
         modules = list_dirs (options.build_dir)
+        if options.split:
+            modules = list_dirs (os.path.join (options.build_dir, options.flat_apply_dir))
         module_re = '|'.join (modules)
 
     # Patches with a simple digit suffix are aggregated into one branch
@@ -393,6 +481,8 @@
 
 def create_gitignores (dir):
     for i in filter_out (operator.not_, gitignores.split ('\n')):
+        if options.split:
+            i = i.replace ('/bootstrap', '/bootstrap/bootstrap')
         if i[0] == '/':
             file (dir + '/.gitignore', 'a').write (i + '\n')
         else:
@@ -430,15 +520,19 @@
         self.milestone = options.milestone
         self.pristine = 'upstream/%(workspace)s-m%(milestone)s' % self.__dict__
         self.commits = {}
+        self.log = {}
         if not os.path.exists (self.dir):
             drink = Setup ().get ('DRINK', 'tea')
-            print >> sys.stderr, 'Unpacking source tree - [ go and have some %(drink)s ] ...' % locals ()
+            info ('Unpacking source tree - [ go and have some %(drink)s ] ...' % locals ())
             system ('cd bin && ./unpack')
-            self.system ('touch unpack')
-        if not os.path.isdir (dir + '/.git'):
             create_gitignores (dir)
+            if options.split:
+                move (self.dir)
+                setup_flat_apply_dir (self.dir)
+            self.system ('touch unpack')
+        if not os.path.isdir (dir + '/.git/refs'):
             drink = Setup ().get ('DRINK')
-            print >> sys.stderr, 'Creating GIT archive - [ go and have some %(drink)s ] ...' % locals ()
+            info ('Creating GIT archive - [ go and have some %(drink)s ] ...' % locals ())
             self.system ('git init')
             svn_revision = get_svn_revision ()
             self.commit ('Initial svn:r%(svn_revision)s unpatched.' % locals ())
@@ -468,17 +562,20 @@
                            .replace ('*', '')
                            .replace (' ', '').split ('\n'))
     def get_log (self, branch=''):
-        return self.pipe ('git log --pretty=oneline %(branch)s --' % locals ())
-    def current_commit (self, branch=''):
-        s = self.get_log ('-1 ' + branch)
-        return s[:s.index (' ')]
+        commit = self.get_current_commit (branch)
+        self.log[commit] = self.log.get (commit, self.pipe ('git log --pretty=oneline %(branch)s --' % locals ()))
+        return self.log[commit]
+    def get_current_commit (self, branch=''):
+        if not branch:
+            branch = 'HEAD'
+        return self.pipe ('git rev-parse %(branch)s' % locals ())[:-1]
     def get_commit (self, patch):
         if not self.commits:
             log = self.get_log (self.patched)
             def grok_log_line (s):
                 m = re.match ('([^ ]+) Apply.*/([^/]+[.](diff|patch))', s)
                 if not m:
-                    print >> sys.stderr, 'Skipping line:%(s)s:' % locals ()
+                    info ('Skipping line:%(s)s:' % locals ())
                     return None, None
                 return m.group (2), m.group (1)
             self.commits = dict (map (grok_log_line, log.split ('\n')[:-2]))
@@ -499,14 +596,14 @@
         if not self.is_on_branch (branch):
             self.system ('git checkout %(branch)s' % locals ())
     def get_current_branch (self):
-        return re.search ('(^|\n)\* (.+)', self.pipe ('git branch')).group (2)
+        return self.pipe ('git symbolic-ref HEAD', raise_on_error=False)[len ('refs/heads/'):-1]
     def is_on_branch (self, branch):
         return branch == self.get_current_branch ()
     def has_branch (self, branch):
         return branch in self.get_branches ()
     def apply_patch (self, branches, patches, patch):
         branch = patch_get_branch (patch)
-        print >> sys.stderr, 'Applying patch[%(branch)s]:' % locals (), patch.name
+        info ('Applying patch[%(branch)s]: ' % locals () + patch.name)
         patched = self.get_current_branch ()
         apply_patch (self.dir, patch.file_name)
         base = os.path.basename (patch.file_name)
@@ -523,7 +620,7 @@
                     self.system ('git rebase %(dependency)s' % locals ())
             else:
                 depend_str = ' '.join (dependencies)
-                self.system ('tg create --force %(branch)s %(depend_str)s' % locals ())
+                self.system ('tg create %(branch)s %(depend_str)s' % locals ())
                 self.system ('git commit -am "topgit branch info %(branch)s"' % locals ())
         else:
             self.checkout (branch)
@@ -539,14 +636,14 @@
         self.system ('git branch -D %(scratch)s' % self.__dict__)
     def pick_patch (self, patch, commit):
         branch = patch_get_branch (patch)
-        print >> sys.stderr, 'Picking patch[%(branch)s]:' % locals (), patch.name
+        info ('Picking patch[%(branch)s]: ' % locals () + patch.name)
         self.system ('git cherry-pick -x %(commit)s' % locals ())
     def add_patch (self, branches, patches, patch):
         if patch.name in self.get_log ():
-            print >> sys.stderr, 'patch already applied, skipping:', patch.name
+            info ('patch already applied, skipping: ' + patch.name)
             return
         if file (patch.file_name).read ().find ('\n+++ ') == -1:
-            print >> sys.stderr, 'patch is empty, skipping:', patch.name
+            info ('patch is empty, skipping: ' + patch.name)
             return
         commit = None
         branch = patch_get_branch (patch)
@@ -564,7 +661,7 @@
     def dump_gob (self, branches, patches, branch):
         gob_dir = self.dir + '/.git/refs/gob'
         if not os.path.exists (gob_dir):
-            os.makedirs (gob_dir)
+            makedirs (gob_dir)
         branch_patches = branches.get (branch, [])
         if not branch_patches:
             return
@@ -579,9 +676,9 @@
         issue_string = ', '.join (issues)
         dependencies = filter (lambda x: x != 'pristine', branch_get_dependencies (branches, patches, branch))
         dependencies_string = ', '.join (dependencies)
-        commit = self.current_commit ()
+        commit = self.get_current_commit ()
         gob_file_name = os.path.join (gob_dir, branch)
-        print >> sys.stderr, 'Writing:', gob_file_name
+        info ('Writing: ' + gob_file_name)
         file (gob_file_name, 'w').write ('''%(commit)s
 state: stable
 issue: %(issue_string)s
@@ -726,10 +823,6 @@
         git = Git (self.options.build_dir, self.options.patched)
     def git_export (self):
         '''export to GIT with branches'''
-        if (not options.force
-            and (self.options.milestone not in ['19', '21', '25']
-                 or self.options.workspace != 'dev300')):
-            raise SystemFailed ('Export only supported for dev300-m19,21,25.  Use --force to override')
         git = Git (self.options.build_dir, self.options.patched, clean=True)
         patches = self.get_patches ()
         branches = self.get_branches ()
@@ -805,6 +898,11 @@
         git = Git (self.options.build_dir, self.options.patched)
         git.system ('git checkout -f %(patched)s' % git.__dict__)
         git.system ('git reset --hard pristine')
+        git.system ('git clean -df')
+        git.system ('rm -r .git/refs/top-bases')
+        cmd = 'xargs git branch -D'
+        if options.split:
+            git.system ('''git branch | grep -Ev '/|master|patched|pristine|upstream|%(patched)s' | xargs tg delete -f ''' % git.__dict__)
         git.system ('''git branch | grep -Ev '/|master|patched|pristine|upstream|%(patched)s' | xargs git branch -D''' % git.__dict__)
     def patch_depend (self):
         '''patch-depend PATCH-1 PATCH-2 - show overlap between patches'''
@@ -859,6 +957,11 @@
         print 'MATCH:', match
         print 'EXTRA:', extra
         print 'INDEPENDENT:', independent
+    def move (self):
+        move (options.build_dir)
+        setup_flat_apply_dir (options.build_dir)
+    def move_back (self):
+        move_back (options.build_dir)
 
 def get_cli_parser ():
     p = optparse.OptionParser ()
@@ -869,6 +972,7 @@
                 if d[k].__doc__ and type (d[k]) == type (lambda x: x)]
     commands.sort ()
 
+    global src_dir
     src_dir = ''
     if not os.path.exists ('patches'):
         src_dir = '../'
@@ -924,6 +1028,7 @@
                   metavar='INT',
                   help='use FUZZ as fuzz factor for patch overlap')
     p.add_option ('--force', action='store_true', dest='force', default=False)
+    p.add_option ('--split', action='store_true', dest='split', default=False)
     p.add_option ('--topgit', action='store_true', dest='topgit', default=False)
     p.add_option ('-v', '--verbose', action='count', dest='verbose', default=1)
     p.add_option ('-q', '--quiet', action='count', dest='quiet', default=0)
@@ -948,7 +1053,9 @@
     sys.exit (2)
 
 def set_option_defaults (options):
+    options.flat_apply_dir = '.git/apply-dir'
     options.verbose -= options.quiet
+    options.apply_dir = options.apply_dir.replace ('ooo300', 'dev300')
     if not options.distros:
         options.distros = ['SUSE']
     if not options.dir_branch:

Added: trunk/bin/modules2.txt
==============================================================================
--- (empty file)
+++ trunk/bin/modules2.txt	Thu Aug 14 12:50:24 2008
@@ -0,0 +1,17 @@
+ure=bridges,cli_ure,codemaker,cppu,cppuhelper,cpputools,idlc,io,javaunohelper,jurt,jut,jvmaccess,jvmfwk,offapi,offuh,pyuno,rdbmaker,registry,remotebridges,ridljar,sal,salhelper,stoc,store,udkapi,unoil,ure,xml2cmp
+sdk=autodoc,cosv,odk,sdk_oo,udm,unodevtools
+base=dbaccess,reportdesign
+calc=sc,scaddins,sccomp,chart2
+l10n=extras,helpcontent2,readlicense_oo
+writer=sw,starmath
+impress=sd,animations,slideshow,sdext
+artwork=default_images,external_images,ooo_custom_images
+filters=binfilter,filter,hwpfilter,unoxml,writerfilter,writerperfect,xmerge,oox
+testing=qadevOOo,smoketestoo_native,testshl2,testtools
+bootstrap=config_office,dmake,instsetoo_native,scp2,solenv,soltools,stlport
+libs_gui=basebmp,basegfx,canvas,comphelper,cppcanvas,dtrans,goodies,i18npool,i18nutil,o3tl,padmin,psprint,psprint_config,regexp,rsc,sax,sot,svtools,toolkit,tools,transex3,ucbhelper,unotools,vcl,vos
+libs_core=avmedia,basic,configmgr,connectivity,desktop,embeddedobj,eventattacher,fileaccess,fpicker,framework,idl,linguistic,officecfg,oovbaapi,sandbox,scripting,sfx2,shell,sj2,svx,sysui,ucb,uui,xmlhelp,xmloff,xmlscript
+libs_extern=afms,agg,beanshell,epm,external,fondu,freetype,hsqldb,jfreereport,libegg,libtextcat,libxmlsec,msfontextract,np_sdk,rhino,sane,twain,lpsolve,icc,openssl,unixODBC,vigra,x11_extensions,xpdf,hyphen,libwpd,lucene,redland,cppunit
+components=accessibility,automation,basctl,bean,crashrep,embedserv,extensions,forms,javainstaller2,lingucomponent,MathMLDTD,package,setup_native,UnoControls,wizards,xmlsecurity
+postprocess=postprocess,packimages
+libs_extern_sys=berkeleydb,bitstream_vera_fonts,expat,icu,jpeg,libxml2,libxslt,moz,neon,python,zlib,saxon,stax,boost,curl,dictionaries,cairo,hunspell

Modified: trunk/bin/src-pack2
==============================================================================
--- trunk/bin/src-pack2	(original)
+++ trunk/bin/src-pack2	Thu Aug 14 12:50:24 2008
@@ -6,58 +6,19 @@
 # don't forget to update it there.
 #
 
-use strict;
+#use strict;
 
 # Towards a more modular build ...
 
 my $tar_opts='cj';
-my %module_map = (
-    'ure' => [ 'bridges', 'cli_ure', 'codemaker', 'cppu', 'cppuhelper',
-	       'cpputools', 'idlc', 'io', 'javaunohelper', 'jurt', 'jut',
-	       'jvmaccess', 'jvmfwk', 'offapi', 'offuh', 'pyuno', 'rdbmaker',
-	       'registry', 'remotebridges', 'ridljar', 'sal', 'salhelper',
-	       'stoc', 'store', 'udkapi', 'unoil', 'ure', 'xml2cmp' ],
-    'sdk' => [ 'autodoc', 'cosv', 'odk', 'sdk_oo', 'udm', 'unodevtools' ],
-    'base' => [ 'dbaccess', 'reportdesign' ],
-    'calc' => [ 'sc', 'scaddins', 'sccomp', 'chart2' ],
-    'l10n' => [ 'extras', 'helpcontent2', 'readlicense_oo' ],
-    'writer' => [ 'sw', 'starmath' ],
-    'impress' => [ 'sd', 'animations', 'slideshow', 'sdext' ],
-    'artwork' => [ 'default_images', 'external_images', 'ooo_custom_images' ],
-    'filters' => [ 'binfilter', 'filter', 'hwpfilter', 'unoxml',
-		   'writerfilter', 'writerperfect', 'xmerge', 'oox' ],
-    'testing' => [ 'qadevOOo', 'smoketestoo_native', 'testshl2', 'testtools' ],
-    'bootstrap' => [ 'config_office', 'dmake', 'instsetoo_native', 'scp2',
-		     'solenv', 'soltools', 'stlport' ],
-    'libs_gui' => [ 'basebmp', 'basegfx', 'canvas', 'comphelper', 'cppcanvas',
-		    'dtrans', 'goodies', 'i18npool', 'i18nutil', 'o3tl',
-		    'padmin', 'psprint', 'psprint_config', 'regexp', 'rsc', 
-		    'sax', 'sot', 'svtools', 'toolkit', 'tools', 'transex3',
-		    'ucbhelper', 'unotools', 'vcl', 'vos' ],
-    'libs_core' => [ 'avmedia', 'basic', 'configmgr', 'connectivity',
-		     'desktop', 'embeddedobj', 'eventattacher', 'fileaccess',
-		     'fpicker', 'framework', 'idl', 'linguistic',
-		     'officecfg', 'oovbaapi', 'sandbox', 'scripting',
-		     'sfx2', 'shell', 'sj2', 'svx', 'sysui',
-		     'ucb', 'uui', 'xmlhelp', 'xmloff', 'xmlscript' ],
-    'libs_extern' => [ 'afms', 'agg', 'beanshell', 'epm', 'external',
-		       'fondu', 'freetype', 'hsqldb', 'jfreereport',
-		       'libegg', 'libtextcat', 'libxmlsec', 'msfontextract',
-		       'np_sdk', 'rhino', 'sane', 'twain', 'lpsolve', 'icc',
-		       'openssl', 'unixODBC', 'vigra', 'x11_extensions',
-		       'xpdf', 'hyphen', 'libwpd', 'lucene', 'redland',
-		       'cppunit' ],
-    'components' => [ 'accessibility', 'automation', 'basctl', 'bean',
-		      'crashrep', 'embedserv', 'extensions', 'forms',
-		      'javainstaller2', 'lingucomponent', 'MathMLDTD',
-		      'package', 'setup_native', 'UnoControls', 'wizards',
-		      'xmlsecurity' ],
-    'postprocess' => [ 'postprocess', 'packimages' ],
-    'libs_extern_sys' => [ 'berkeleydb', 'bitstream_vera_fonts', 'expat', 'icu', 
-			   'jpeg', 'libxml2', 'libxslt', 'moz', 'neon',
-			   'python', 'zlib', 'saxon', 'stax', 'boost', 'curl',
-			   'dictionaries', 'cairo', 'hunspell' ]
-);
+
+$0 =~ /(.*)\//;
+my $dir = $1;
+my $module_list = "$dir/modules2.txt";
+open MODULES, "$module_list" or die die "Can't open: $module_list: $!";
+my %module_map = map {/(.*)=(.*)$/; ($1, [split /,/, $2])} <MODULES> or die;
+# /})];
+
 my $force = 0;
 
 my $src;
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]