path: root/dh_illumos_gate

#!/usr/bin/perl -w

=head1 NAME

dh_illumos_gate - unpack and prepare illumos sources

=cut

use strict;
use Debian::Debhelper::Dh_Lib;
use Dpkg::Changelog::Parse;
use Cwd;

=head1 SYNOPSIS

B<dh_illumos_gate> [S<I<debhelper options>>] [options] [-- [tar options]]
[files and/or directories to pack/unpack]

=head1 DESCRIPTION

C<dh_illumos_gate> is responsible for unpacking illumos sources and adjusting the sources
for Debian build environment (paths to programs, command options, linker and compiler flags, etc.).

C<dh_illumos_gate> unpacks specified files or directories, if any.
Otherwise it unpacks the entire F<illumos-gate/usr/>.
In any case B<-X> option can be used to exclude some files or directories,
and top directory F<illumos-gate> is always stripped.

If files interested to C<dh_illumos_gate> are unpacked they will be configured.
See L</DETAILS>.

After executing C<dh_illumos_gate> one can use commands like this to configure or build
components of illumos gate:

    ksh93 usr/bldenv.sh usr/env.sh -c "cd usr/src/lib/libzfs && make --sun install"

L<dh_illumos_make(1)> provides more convenient way to do this.

=head1 OPTIONS

=over 4

=item B<--build>

Unpack or pack (by B<--create-orig>) files known to be required for building illumos components.
These are F<usr/src/tools/scripts/bldenv.sh>, F<usr/src/Makefile*>,
F<usr/src/common/mapfiles/*>. If any file under F<usr/src/lib> is
requested, F<usr/src/lib/Makefile*> are [un]packed too. Same for
F<usr/src/cmd>. This option is provided only for convenience.
You are free to pack and unpack these files explicitly.

=item B<-X>I<PATTERN> B<--exclude=>I<PATTERN>

Do not pack or unpack files or directories matching I<PATTERN>.
This option can be used multiple times. I<PATTERN> is passed
as is to L<tar(1)> via I<--exclude=PATTERN>.

=item B<--destdir=>I<destdir>

Destination directory for B<--configure>, B<--checkout> and unpacking (default action).

=item B<--create-orig>

Create I<.orig> tarball out of the listed files and directories.
Note that part of path before F<usr/src> will be stripped.

=item B<--configure>

Copy existing sources (F<./usr>) into the build directory specified by
B<--destdir> (instead of unpacking the illumos-gate tarball)
and then configure it.

=item B<--unpack-only>

Only unpack illumos-gate tarball, do not configure.

=item B<--checkout>[I<=git-hash>]

Clone illumos-gate git repository and checkout I<git-hash> in there.
If not specified, hash is extracted from topmost version in F<debian/changelog>:
C<1.2.3.git.<hash>-4>.
Could be used for B<--create-orig>. Repository is https://github.com/illumos/illumos-gate.git,
unless specified by B<--repository>. You should also set B<--destdir>.

=item B<--repository=>I<URI>

illumos-gate git repository. Default is https://github.com/illumos/illumos-gate.git.

=item B<--root=>I<path>

Defines root directory where build result will be installed: executables, libraries, headers, etc.
Defaults to F<./debian/tmp>. This can be overrided by L<dh_illumos_make(1)>.
If absent, it will be created.

=back

=head1 EXAMPLES

=over 4

=item Unpack and configure the whole illumos source tree (F<usr/>)

    dh_illumos_gate

=item Unpack and configure whole illumos source tree, but F<usr/src/uts/>

    dh_illumos_gate -X usr/src/uts

=item Unpack and configure whole illumos source tree, but F<usr/src/uts/>;
then add F<usr/src/uts/common/sys/>

    dh_illumos_gate -X usr/src/uts
    dh_illumos_gate usr/src/uts/common/sys

=item Unpack all Makefiles under F<usr/src/>

    dh_illumos_gate usr/src/Makefile\*

=item Unpack all header files under F<usr/src/uts>

    dh_illumos_gate -- --wildcards-match-slash usr/src/uts/*.h

=item Create source tarball out of F<usr/src/cmd> excluding F<*.xml> files

    dh_illumos_gate --create-orig [path/to/]usr/src/cmd -X*.xml

=item Prepare for debug and non-debug builds

    dh_illumos_gate --configure --destdir=workdir/debug
    dh_illumos_gate --configure --destdir=workdir/non-debug

=item Migrate to 3.0 (quilt) format

    dh_illumos_gate --unpack-only files...
    dh_illumos_gate --create-orig files...

=item Create new orig tarball

    dh_illumos_gate --checkout=a5723... --destdir=foo
    dh_illumos_gate --create-orig foo/usr/src

=back

=head1 FILES

=over 4

=item /usr/src/illumos-gate/illumos-gate-$(VER).tar.xz

This file contains complete and unmodified illumos sources.
It is provided by package I<illumos-source-$(VER)>.

Version C<$(VER)> is determined from F<debian/changelog>:

    (\d+\.\d+).* => \1, e. g.:
    2.10+3       => VER = 2.10

=back

=head1 DETAILS

Here are things which this helper does after unpacking illumos sources.

=over 4

=cut

$dh{ROOT}       = 'debian/tmp';
$dh{CHECKOUT}   = undef;
$dh{DESTDIR}    = '.';
$dh{REPOSITORY} = 'https://github.com/illumos/illumos-gate.git';

init(
    options => {
        'build'        => \$dh{BUILD},
        'checkout:s'   => \$dh{CHECKOUT},
        'configure'    => \$dh{CONFIGURE},
        'create-orig'  => \$dh{CREATE_ORIG},
        'destdir=s'    => \$dh{DESTDIR},
        'repository=s' => \$dh{REPOSITORY},
        'root=s'       => \$dh{ROOT},
        'unpack-only'  => \$dh{UNPACK_ONLY},
    }
);

my %options   = ( file => 'debian/changelog' );
my $changelog = changelog_parse(%options);
my $version   = $changelog->{'Version'};
my $cwd       = getcwd();

my @tar_X = ();
if ( defined( $dh{'EXCLUDE'} ) && $dh{'EXCLUDE'} ) {
    foreach my $x ( @{ $dh{'EXCLUDE'} } ) {
        push @tar_X, '--exclude';
        push @tar_X, $x;
    }
}

if ( defined $dh{CHECKOUT} ) {
    if ( -e $dh{DESTDIR} ) {
        error("Destination path `$dh{DESTDIR}' exists.");
    }
    if ( !$dh{CHECKOUT} ) {
        $dh{CHECKOUT} = $version =~ s!.*\Wgit\W?(\w+)-[^-]+$!$1!r;
    }
    doit( 'git', 'clone', '-n',         $dh{REPOSITORY}, $dh{DESTDIR} );
    doit( 'git', '-C',    $dh{DESTDIR}, 'checkout',      $dh{CHECKOUT} );
    exit;
}
elsif ( $dh{CREATE_ORIG} ) {
    my $source         = $changelog->{'Source'};
    my $source_version = $version =~ s!^(.*)-[^-]+$!$1!r;
    my $tarball        = "$cwd/../${source}_${source_version}.orig.tar.xz";
    my $transform      = "s,.*usr/src,$source-$source_version/usr/src,";
    my @tar            = (
        'tar',                   '-c',
        '-J',                    '--exclude-vcs',
        '--exclude-vcs-ignores', '--owner=0',
        '--group=0',             '--mode',
        'u=rwX,go=rX',           '--transform',
        "'$transform'",          '-f',
        "'$tarball'",            @tar_X
    );
    my @extra = ();
    if ( $dh{'BUILD'} ) {
        my $r = $ARGV[0] =~ s!(.*)usr/src.*!$1!r;    # XXX Sloppy
        push @extra,
          (
            "${r}usr/src/Makefile*",
            "${r}usr/src/common/mapfiles",
            "${r}usr/src/tools/scripts/bldenv.sh",
          );
        if ( grep { m|(.*)usr/src/lib/.*| } @ARGV ) {
            push @extra, "$1usr/src/lib/Makefile*";
        }
        if ( grep { m|(.*)usr/src/cmd/.*| } @ARGV ) {
            push @extra, "$1usr/src/cmd/Makefile*";
        }
    }

    # We need shell wildcards here, thus complex_doit, not doit
    # sort - to drop dublicates (lead to hardlinks in the tarball):
    complex_doit("find @ARGV @extra -type f | sort -u | @tar -T -");
    exit;
}
elsif ( $dh{CONFIGURE} ) {
    if ( -e $dh{DESTDIR} ) {
        error("Destination path `$dh{DESTDIR}' exists.");
    }
    doit( 'mkdir', '-p', $dh{DESTDIR} );
    doit( 'cp', '-a', 'usr', "$dh{DESTDIR}/usr" );

    # fall through
}
else {
    # Default is to unpack for backward compatibility

    my $ver;
    if ( $version =~ /^(\d+(\.\d+)+(\.git\w+)?).*$/ ) {
        $ver = $1;
    }
    else {
        error(  "Could not determine illumos source version from changelog. "
              . "It should be like 5.10.6.gitXYZ+1, but it is '$version'" );
    }
    my $tarball = "/usr/src/illumos-gate/illumos-gate-$ver.tar.xz";

    if ( !-f $tarball ) {
        error(
"$tarball does not exist. You might not have package illumos-source-$ver "
              . "installed or have a typo in debian/changelog" );
    }

    my @to_extract = ();
    if (@ARGV) {
        if ( $dh{'BUILD'} ) {
            push @to_extract,
              (
                'usr/src/Makefile*', 'usr/src/common/mapfiles',
                'usr/src/tools/scripts/bldenv.sh',
              );
            if ( grep { m|.*usr/src/lib/.*| } @ARGV ) {
                push @to_extract, 'usr/src/lib/Makefile*';
            }
            if ( grep { m|.*usr/src/cmd/.*| } @ARGV ) {
                push @to_extract, 'usr/src/cmd/Makefile*';
            }
        }
        push @to_extract, map { m,(?:illumos-gate/)?(.*),; $1 } @ARGV;
    }
    else {
        push @to_extract, 'usr';
    }

    my @to_extract_real = ();
    foreach my $p (@to_extract) {
        if ( $p =~ /^-/ ) {
            push @to_extract_real, $p;    #XXX: tar option
        }
        else {
            push @to_extract_real, 'illumos-gate/' . $p;    #XXX: path
        }
    }

    doit( 'mkdir', '-p', $dh{DESTDIR} );
    doit( 'tar', '-C', $dh{DESTDIR}, '-x', '-f', $tarball, @tar_X,
        '--wildcards', '--strip-components=1', @to_extract_real );

    if ( $dh{UNPACK_ONLY} ) {
        exit;
    }
}

=item Saving original files

Any files from illumos sources, changed by this helper,
are saved with F<.orig> suffix, so you can investigate changes. For example,
you can use C<diff -dub usr/src/Makefile.master.orig usr/src/Makefile.master>
to see all changes.

=cut

=item Changes to F<usr/src/Makefile.master> (if exists):

Remove option C<-s> (strip) from install command for directories and files.
This options does not make sense for directories and GNU L<install(1)> is
not tolerant here. Also replace option C<-f> with C<-t>.

Append C<i386_XARCH> with C<-m32> since native host compiler can produce 64-bit
code by default.

Set initial value of C<POST_PROCESS_O> to C<:> since it may be appended
later with other commands separated by C<;>, but bash and dash
do not like stand-alone C<;>. Korn shell is ok.

Explicitly undefine some CPP macros related to amd64 for building 32-bit
objects, e. i. options like C<-U__amd64> are appended to C<i386_XARCH>
and C<i386_AS_XARCH>.

C<i386_AS_XARCH> flags are appended to C<COMPILE.s> command.

Delete (set to empty) variable C<MAPFILE.LEX>, cause we are not
using shared lex library (F<libl.so.1>), but static Flex
library (libl.a).

In variables C<OFFSETS_CREATE> and C<OFFSETS_CREATE64> replace
variables C<$(CTFSTABS)> and C<$(CTFCONVERT)> with explicit
F<ctfstabs> and F<ctfconvert>, because those variables are
empty when building without CTF, but creating offsets
really requires CTF data. This is used when compiling libc and
illumos kernel.

=cut

my $multiarch = `dpkg-architecture -qDEB_HOST_MULTIARCH`;
chomp $multiarch;
my $bits = `dpkg-architecture -qDEB_HOST_ARCH_BITS`;
chomp $bits;

# In illumos source 32-bit binaries are default
my ( $libdir32, $libdir64, $usrlibdir32, $usrlibdir64 );

if ( $bits == 64 ) {
    $libdir32    = "lib32";
    $usrlibdir32 = "usr/$libdir32";
    $libdir64    = "lib/$multiarch";
    $usrlibdir64 = "usr/$libdir64";
}
else {
    $libdir32    = "lib/$multiarch";
    $usrlibdir32 = "usr/$libdir32";
    $libdir64    = "lib64";
    $usrlibdir64 = "usr/$libdir64";
}

my $Makefile_master      = $dh{DESTDIR} . '/usr/src/Makefile.master';
my $Makefile_master_orig = $Makefile_master . '.orig';
if ( -f $Makefile_master ) {
    if ( !-e $Makefile_master_orig ) {
        doit( 'cp', '-f', $Makefile_master, $Makefile_master_orig );
    }

    # Be careful with modern CPP:
    my $i386_undef =
      '-Uamd64 -U__amd64 -U__amd64__ -U__x86_64__ -U__x86_64 -U_LP64';
    doit(
        'sed', '-r', '-i', '
        /^INS\.(file|dir)/ s, -s,,;
        s/^POST_PROCESS_O\s*=\s*$/POST_PROCESS_O = :/;
        /^INS\.file/ s, -f, -t,;
        /^i386_XARCH\s*=/s,$, -m32 ' . $i386_undef . ',;
        /^i386_AS_XARCH\s*=/s,$, ' . $i386_undef . ',;
        /^COMPILE\.s/s,$, $($(MACH)_AS_XARCH),;
        /^MAPFILE.LEX/d;
        /^OFFSETS_CREATE/ s,\$\(CTFSTABS\),ctfstabs,;
        /^OFFSETS_CREATE/ s,\$\(CTFCONVERT\),ctfconvert,;
        ', $Makefile_master
    );
}

=item Changes F<usr/src/lib/Makefile.lib> (if exists):

Append C<CFLAGS> to C<BUILD.SO> command, because C<BUILD.SO>
by default uses C compiler to create shared library.

Set C<ROOTLIBDIR>, C<ROOTLIBDIR64>, C<ROOTFS_LIBDIR> and C<ROOTLIBDIR64>
to match Debian multiarch layout. For example C<ROOTLIBDIR64> =
F</usr/lib/x86_64-illumos> and C<ROOTFS_LIBDIR> = F</lib32> on I<illumos-amd64>.
It cannot be done in environment file F<env.sh> (see below), because
in some cases C<ROOTLIBDIR> will be overriden by C<ROOTFS_LIBDIR> within makefiles.

=cut

my $Makefile_lib      = $dh{DESTDIR} . '/usr/src/lib/Makefile.lib';
my $Makefile_lib_orig = $Makefile_lib . '.orig';
if ( -f $Makefile_lib ) {
    if ( !-e $Makefile_lib_orig ) {
        doit( 'cp', '-f', $Makefile_lib, $Makefile_lib_orig );
    }
    doit(
        'sed', '-r', '-i', '
        /^BUILD.SO/ s,\$\(CC\),$(CC) $(CFLAGS),;
        ', $Makefile_lib
    );
    doit(
        'sed', '-r', '-i', "
        s,^(ROOTFS_LIBDIR)\\s*=.*,\\1=\$(ROOT)/$libdir32,;
        s,^(ROOTFS_LIBDIR64)\\s*=.*,\\1=\$(ROOT)/$libdir64,;
        s,^(ROOTLIBDIR)\\s*=.*,\\1=\$(ROOT)/$usrlibdir32,;
        s,^(ROOTLIBDIR64)\\s*=.*,\\1=\$(ROOT)/$usrlibdir64,;
        ", $Makefile_lib
    );
}

=item Changes F<usr/src/cmd/Makefile.cmd> (if exists):

Remove trailing spaces in the line C<TEXT_DOMAIN=SUNW_OST_OSCMD>
Maybe we should patch GNU make to do it for us in SunOS mode.

Remove linker flag for setting ELF interpreter (-Wl,-I/lib/ld.so.1)

=cut

my $Makefile_cmd      = $dh{DESTDIR} . '/usr/src/cmd/Makefile.cmd';
my $Makefile_cmd_orig = $Makefile_cmd . '.orig';
if ( -f $Makefile_cmd ) {
    if ( !-e $Makefile_cmd_orig ) {
        doit( 'cp', '-f', $Makefile_cmd, $Makefile_cmd_orig );
    }
    doit(
        'sed', '-r', '-i', '
        /^TEXT_DOMAIN/ s,\s+$,,g;
        ', $Makefile_cmd
    );
    doit(
        'sed', '-r', '-i', '
        /LDFLAGS\s*\+=\s*-Wl,-I\/lib\/ld\.so\.1/d;
        ', $Makefile_cmd
    );
}

=item Creates F<usr/env.sh>

Set C<CODEMGR_WS> to the B<--destdir>.

Set C<VERSION> from F<./debian/changelog>.
For example it can be C<2.10-3>.

Set C<LD_ALTEXEC>=F</usr/bin/ld-gnu-to-sun> to call L<sunld(1)>
instead of GNU L<ld(1)>. Default system GCC is build to use GNU L<ld(1)>,
but L<sunld(1)> is required to build most illumos components.
See L<ld-gnu-to-sun(1)>.

Set proto area to be F<C<CODEMGR_WS>/root>.

Disable linting libraries by replacing C<LINT> with ":".
So it will do nothing. Successfully. Also define C<LINTLIB> to empty
string to prevent installation of lint libraries (which do not exist).

Disable modifying comment section of ELFs by replacing C<MCS> with ":".

Disable stripping by replacing C<STRIP> with ":".

Enable GCC and GCC4 mode by defining C<__GNUC> and C<__GNUC4>.

Change GCC root to F</usr>, so the default system compiler is used.

Define C<LEX> to be C</usr/bin/flex -l>.

Export all C<DEB_*> variables (using C<dpkg-architecture -s>)
to make them available from makefiles.

Unsets C<CPPFLAGS>, C<CFLAGS>, C<LDFLAGS>.
Many related variables are unset by F<bldenv.sh>,
but these ones. They break build.

=cut

chdir $dh{DESTDIR};
my $codemgr_ws = getcwd();
chdir $cwd;

doit( 'mkdir', '-p', $dh{ROOT} );
chdir $dh{ROOT};
my $root = getcwd();
chdir $cwd;

my $env  = $dh{DESTDIR} . '/usr/env.sh';
my $mach = `uname -p`;
chomp $mach;
if ( open( ENV, '>', $env ) ) {
    print ENV "# This file was generated by dh_illumos_gate(1)\n";
    print ENV "export VERSION='$version'\n";
    print ENV "export CODEMGR_WS='$codemgr_ws'\n";
    print ENV "export ROOT='$root'\n";
    print ENV "export SRC='$codemgr_ws/usr/src'\n";
    print ENV "export MULTI_PROTO='no'\n";
    print ENV "export CW_NO_SHADOW=1\n";
    print ENV "export LD_ALTEXEC='/usr/bin/ld-gnu-to-sun'\n";
    print ENV "export BUILD_TOOLS='/opt'\n";
    print ENV "export MACH='$mach'\n";
    print ENV "export LINT=:\n";
    print ENV "export LINTLIB=''\n";
    print ENV "export __GNUC=''\n";
    print ENV "export __GNUC4=''\n";
    print ENV "export GCC_ROOT='/usr'\n";
    print ENV "export CW_GCC=gcc\n";
    print ENV "export SAVEARGS=\n";
    print ENV "export LEX='/usr/bin/flex -l'\n";
    print ENV "export STRIP=':'\n";
    print ENV "export MCS=':'\n";
    print ENV "export DEB_LIBDIR_32='/$libdir32'\n";
    print ENV "export DEB_USRLIBDIR_32='/$usrlibdir32'\n";
    print ENV "export DEB_LIBDIR_64='/$libdir64'\n";
    print ENV "export DEB_USRLIBDIR_64='/$usrlibdir64'\n";
    print ENV "unset CFLAGS\n";
    print ENV "unset CPPFLAGS\n";
    print ENV "unset LDFLAGS\n";
    print ENV
"export LDLIBS32=\"-L\$ROOT/\$DEB_LIBDIR_32 -L\$ROOT/\$DEB_USRLIBDIR_32\"\n";
    print ENV
"export LDLIBS64=\"-L\$ROOT/\$DEB_LIBDIR_64 -L\$ROOT/\$DEB_USRLIBDIR_64\"\n";
    print ENV 'export LDLIBS_NATIVE="',
      ( $bits == 64 ) ? '$LDLIBS64' : '$LDLIBS32', "\"\n";
    print ENV `dpkg-architecture -s`;

    close(ENV);
}
else {
    error("Failed to write `$env': $!");
}

=item Create F<usr/bldenv.sh> (if F<usr/src/tools/scripts/bldenv.sh> exists)

Copy F<usr/src/tools/scripts/bldenv.sh>
to F<usr/bldenv.sh>.

Fix definition of C<CTF*> variables, so this script will
not override them if they are already defined. These variables
can be set to ":", when executing L<dh_illumos_make(1)>
with option C<--without-ctf>.

=cut

my $bldenv_sh_orig = $dh{DESTDIR} . '/usr/src/tools/scripts/bldenv.sh';
my $bldenv_sh      = $dh{DESTDIR} . '/usr/bldenv.sh';

if ( -f $bldenv_sh_orig ) {
    doit( 'cp', '-f', $bldenv_sh_orig, $bldenv_sh );
    doit(
        'sed', '-r', '-i', '
        s,export +(CTF.+)="(.+)",export \1="${\1:-\2}",;
        s,^PATH=".+",PATH="${PATH}:/opt/onbld/bin:/opt/onbld/bin/${MACH}:.",;
        ', $bldenv_sh
    );
}

=back

=head1 NOTES

To clean package (via C<./debian/rules clean>) you should just remove F<usr/> directory,
or the directory you specified by B<--destdir>:

    rm -rf usr


C<dh_illumos_gate --build> will fail when it's unpacking files which should be unpacked
due to option C<--build>. This is a L<tar(1)> issue (tar: ...: Not found in archive).
For example, this will fail:

    dh_illumos_gate --build usr/src/cmd usr/src/lib/Makefile.lib

Workaround:

    dh_illumos_gate --build usr/src/cmd
    dh_illumos_gate usr/src/lib/Makefile.lib

=head1 SEE ALSO

L<debhelper(7)>, L<dh_illumos_make(1)>.

=head1 AUTHOR

Igor Pashev <pashev.igor@gmail.com>

=cut