path: root/install

#!/usr/bin/python

"""
Written by Igor Pashev <pashev.igor@gmail.com>

The author has placed this work in the Public Domain,
thereby relinquishing all copyrights. Everyone is free
to use, modify, republish, sell or give away this work
without prior consent from anybody.
"""
from __future__ import print_function

from DysonInstaller.hdd import HDD
from DysonInstaller.snack import *
from snack import *
from subprocess import Popen, PIPE, call
from pprint import pprint
from tempfile import mkstemp, mkdtemp
from urllib2 import urlopen, URLError, HTTPError
from time import sleep
from fnmatch import fnmatch
from os.path import basename, exists
import os
import re
import sys
import math
import shutil
import traceback
import apt

# Snack screen
screen = None

# Physical network interfaces
physlinks = []

# List of disks installed on the system:
hdds = []

# List of zpools found on system:
zpools = []

# List of imported zpools, we should export them after installation:
imported_zpools = []

# Name of the root ZFS pool: either existing or newly created
rpool = None

# Name of the root slice, where ZFS root pool will be created (e. g. c0t0d0s0)
# Used for installing GRUB
rslice = None

# Object, describing solaris partition
solaris_partition = None

# XXX rslice and solaris_partition aare known only
# on new clean installations, so we allow installing GRUB
# only on new installations

# Created boot environment, e. g. rpoot/ROOT/osdy-0
bootenv = None

# Install root dir, mount point for bootenv
rootdir = None

# Version to install:
# XXX this line will be replaced by protostar
codename = 'lacaille'

# Arch to install:
# XXX Use dpkg-architecture when we support more ;-)
archname = 'illumos-amd64'

# Dyson mirrors:
mirror = None
mirrors = [
    ('http://ftp.nluug.nl/os/illumos/dyson/apt/', 'Netherlands'),
    ('http://mirror-us.osdyson.org/apt/', 'USA'),
    ('http://apt.osdyson.org', 'Japan'),
    ]

apt_cache = None # APT cache, used to check if package exists, updated after 'apt-get update'
profile = None
profiles = [
    {
        # This must be the first
        'name': 'Minimal',
        'desc' : 'Minimal package set with minimal configuration',
    },
    {
        'name': 'Basic',
        'desc' : 'Basic system without X',
        # Some packages may be absent, they will be skipped:
        'packages' : '''locales vim bash-completion mc sudo lynx
            mutt wget curl openssh-server inetutils-ping inetutils-traceroute host less
            ntp
            '''.split(),
    },
    {
        'name': 'Desktop',
        'desc' : 'Install X, XFCE and some graphical applications',
        'packages' : '''mc locales bash-completion vim-gtk synaptic abiword dillo gnumeric
            sylpheed geany lightdm pidgin qpdfview sudo gimp midori stardict mirage
            xorg xfce4 xfce4-terminal dbus curl wget transmission-gtk xarchiver
            xfce4-screenshooter xfce4-clipman-plugin gksu libgtk2-perl xfce4-xkb-plugin
            gnupg-agent openssh-client inetutils-ping inetutils-traceroute host less
            xfonts-terminus ntp
            '''.split(),
    },
]

# Top of /rpool/boot/grub/menu.lst, if creating new file:
grub_top = r'''
foreground 343434
background F7FbFF
default 0
timeout 5
'''

# Entry in /rpool/boot/grub/menu.lst for newly created boot environment
# This string must be formatted.
# 0 is the partition number,
# a is the slice number; both are not used
grub_entry = r'''
title Dyson at {bootenv}
findroot (pool_{rpool},{partition},{slice})
bootfs {bootenv}
kernel$ /platform/i86pc/kernel/amd64/unix -B $ZFS-BOOTFS
module$ /platform/i86pc/amd64/boot_archive
'''


# Stages of debootstraping in order. Debootstrap gives
# progress for each stage, but we need entire progress.
# Format: mile : (a, z), where:
# - mile is a stage name, e. g. DOWNDEBS;
# - a, z - the start and the end position on progress bar,
#          e. i. a=0 z=100 - entire progress bar (100%);
miles = {
        'START'       : (0, 0),    # Dummy
        'DOWNREL'     : (0, 1),    # Downloading the 'Release' file
        'DOWNPKGS'    : (1, 5),    # Downloading the 'Packages' file
        'SIZEDEBS'    : (5, 10),   # Finding packags sizes
        'DOWNDEBS'    : (10, 25),  # Downloading packages (*.deb)
        'EXTRACTPKGS' : (25, 45),  # Extracting core packages
        'INSTCORE'    : (45, 50),  # Installing core packages
        'UNPACKREQ'   : (50, 60),  # Unpacking required packages
        'CONFREQ'     : (60, 70),  # Configuring required packages
        'UNPACKBASE'  : (70, 85),  # Unpacking the base system
        'CONFBASE'    : (85, 100), # Configuring the base system
}

# FS to be created in boot environment.
# Order is important.
befs = [
    {'name':'/usr', 'options':{}},
    {'name':'/usr/local', 'options':{
                                'compression':'on',
                                }},
    {'name':'/usr/src', 'options':{
                                'compression':'on',
                                }},
    {'name':'/var', 'options':{}},
    {'name':'/var/cache', 'options':{}},
    {'name':'/var/lib', 'options':{}},
    {'name':'/var/lib/dpkg', 'options':{
                                'compression' : 'on',
                                'setuid'      : 'off',
                                'devices'     : 'off',
                                }},
    {'name':'/var/log', 'options':{
                                'compression' : 'on',
                                'setuid'      : 'off',
                                'devices'     : 'off',
                                }},
    {'name':'/var/mail', 'options':{
                                'compression' : 'on',
                                'setuid'      : 'off',
                                'devices'     : 'off',
                                }},
    {'name':'/var/spool', 'options':{
                                'compression' : 'on',
                                'setuid'      : 'off',
                                'devices'     : 'off',
                                }},
    {'name':'/var/tmp', 'options':{
                                'compression' : 'on',
                                'setuid'      : 'off',
                                'devices'     : 'off',
                                }},
    ]

# XXX Not yet.
befs = []

class Abort(Exception):
    ''' User clicked "Cancel" '''
    pass
class NoDisks(Exception):
    ''' No Hard Drives found '''
    pass

def welcome():
    welcome = ButtonChoiceWindow(screen, "Welcome to the Dyson installer",
            '''
Dyson is an operating system derived from Debian and based on Illumos core. \
It uses Illumos unix kernel and libc, ZFS file system and SMF to manage system startup. \
To learn more visit http://osdyson.org

This installer will guide you through disk paritioning, filesystem creation, \
installation of base system and minimal configuration.

Please note, this system IS VERY EXPERIMENTAL. \
You can LOOSE ALL YOUR DATA which this system can reach ;-)

Whould you like to continue?''',
            buttons=[('Install Dyson', True), ('Exit the installer', False)], width=70)
    if not welcome:
        raise Abort('Installation canceled')

def get_memory_size():
    return os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE")


def get_zpool_slice(pool_name):
    result = []
    ''' Returns slices/disks (e. g. c0t4d0s0) on which zpool exists '''
    zpool_cmd = Popen(['zpool', 'status', pool_name], stderr=PIPE, stdout=PIPE)

    # Skip till the header:
    re_header = re.compile(r'NAME\s+STATE\s+READ\s+WRITE\s+CKSUM')
    for line in zpool_cmd.stdout:
        if re_header.search(line):
            break

    re_disk = re.compile(r'(\w+)\s+ONLINE(\s+\d+){3}')
    for line in zpool_cmd.stdout:
        m = re_disk.search(line)
        if m and not m.group(1) in [pool_name, 'mirror']:
            result.append(m.group(1))
    return result

def find_zpools():
    global zpools
    progress = ProgressMessage(screen, title='Please wait', text='Searching for ZFS pools ...')
    zpools = []
    zpool_cmd = Popen(['zpool', 'list', '-H', '-o', 'name'], stdout=PIPE, stderr=PIPE)
    for line in zpool_cmd.stdout:
        zpools.append(line.rstrip())
    zpool_cmd = Popen("zpool import | awk '/pool:/ {print $2}'", shell=True, stdout=PIPE, stderr=PIPE)
    for line in zpool_cmd.stdout:
        zpools.append(line.rstrip())

def find_hdds():
    global hdds
    progress = ProgressMessage(screen, title='Please wait', text='Searching for disks ...')
    hdds = []
    pat = re.compile('\d+\.\s+(\S+)\s+<(.+?)(\s+cyl.*)?>')
    format_cmd = Popen('format </dev/null', shell=True, stdout=PIPE)
    for line in format_cmd.stdout:
        m = pat.search(line)
        if m:
            name = m.group(1)
            desc = m.group(2)
            hdds.append(HDD(name, desc))
    if len(hdds) == 0:
        raise NoDisks('No disks found')


def choose_rpool():
    zpool_items = map(lambda x: (x, x), zpools)
    zpool_items.append(('Create new ZFS pool', None))

    choice = None

    buttons = [('Ok', 'ok'), ('Cancel', 'cancel')]

    while not choice in ['ok', 'cancel']:
        (choice, pool) = ListboxChoiceWindow(screen,
                title='Choose ZFS pool',
                text='Choose ZFS pool where Dyson will be installed.\n'
                'You can selected one of the existing pools or create a new one.',
                items=zpool_items,
                buttons=buttons,
                width=76,
                scroll=1,
                default=0,
                )
        # Enter is pressed:
        if choice == None:
            choice = 'ok'
    if choice == 'ok':
        return pool
    raise Abort('Pool choosing')

def create_root_slice(hdd_name):
    '''Create root slice (0) over entire Solaris partition'''
    progress = ProgressMessage(screen, title='Please wait',
            text='Formatting Solaris partition on disk "{}" ...'.format(hdd_name))

    fd, path = mkstemp()
    os.write(fd, 'partition\nmodify\n1\nyes\n0\n\n\n\n\n\n\nyes\n"{label}"\nquit\nquit\n'.format(label=hdd_name))
    os.close(fd)
    format_cmd = Popen(['format', '-f', path, '-d', hdd_name], stderr=PIPE, stdout=PIPE)
    out, err = format_cmd.communicate()
    os.unlink(path)
    if format_cmd.returncode != 0:
        ButtonChoiceWindow(screen, title='Creating of root slice failed',
                text=err, buttons = ['Ok'], width=70)
        return None
    return hdd_name + 's0'

def create_rpool(hdd):
    global rslice
    rslice = create_root_slice(hdd.name)
    if not rslice:
        return None
    pool_name = None
    for pool_no in ['', 1, 2, 3, 4, 5, 6, 7]:
        pool_name = 'rpool{}'.format(pool_no)
        if not pool_name in zpools:
            break
        else:
            pool_name = None;

    progress = ProgressMessage(screen, title='Please wait',
            text='Creating ZFS pool "{rpool}" on slice "{slice}" ...'.format(rpool=pool_name, slice=rslice))

    zpool_cmd = Popen(['zpool', 'create', '-R', '/mnt/'+pool_name,
        '-m', '/' + pool_name,  '-f', pool_name, rslice],
            stderr=PIPE, stdout=PIPE)
    out, err = zpool_cmd.communicate()
    if 0 != zpool_cmd.returncode:
        ButtonChoiceWindow(screen,
                title='Creating of ZFS root pool failed',
                text=err, buttons = ['Ok'], width=70)
        return None
    imported_zpools.append(pool_name)
    return pool_name

def pool_is_imported(pool):
    rpool_is_imported = call(['zpool', 'status', pool], stdout=PIPE, stderr=PIPE)
    return rpool_is_imported == 0

def configure_rpool():
    global rpool
    find_zpools()
    while not rpool:
        if len(zpools) > 0:
            rpool = choose_rpool()
        if not rpool:
            hdd = choose_hdd(len(zpools))
            if hdd:
                rpool = create_rpool(hdd)
        if rpool:
            if not pool_is_imported(rpool):
                progress = ProgressMessage(screen, title='Please wait',
                        text='Importing ZFS pool "{}" ...'.format(rpool))
                zpool_cmd = Popen(['zpool', 'import', '-R', '/mnt/'+rpool, '-f', rpool], stderr=PIPE, stdout=PIPE)
                out, err = zpool_cmd.communicate()
                if 0 !=  zpool_cmd.returncode:
                    ButtonChoiceWindow(screen,
                            title='Importing of ZFS pool "{}" failed'.format(rpool),
                            text=err, buttons=['Ok'], width=70)
                    rpool = None
                imported_zpools.append(rpool)

def zfs_exists(fs):
    fs_exists = call(['zfs', 'list', '-H', fs], stdout=PIPE, stderr=PIPE)
    return fs_exists == 0

def zfs_create(fs, zvol=None, options={}):
    args = ['zfs', 'create']
    if zvol:
        args += ['-V', zvol]

    for o in options:
        args += ['-o', o + '=' + options[o]]

    args.append(fs)

    zfs_cmd = Popen(args, stderr=PIPE, stdout=PIPE)
    out, err = zfs_cmd.communicate()
    if 0 != zfs_cmd.returncode:
        ButtonChoiceWindow(screen,
                title='Creating of ZFS filesystem failed',
                text=err, buttons = ['Ok'], width=70)
        return False
    return True


def configure_zfs():
    global bootenv
    global rootdir
    progress = ProgressBar(screen,
            title='Creating ZFS filesystems',
            top = 4 + len(befs), # 4 for ROOT, /swap, /home, and BE
            width = 70,
            )
    root = rpool + '/ROOT'
    if not zfs_exists(root):
        progress.text = 'Creating ' + root
        zfs_create(root, options={'canmount':'off', 'mountpoint':'none'})
    progress.advance()

    swap = rpool + '/swap'
    if not zfs_exists(swap):
        progress.text = 'Creating ' + swap
        zfs_create(swap, zvol='2G')
    progress.advance()

    home = rpool + '/home'
    if not zfs_exists(home):
        progress.text = 'Creating ' + home
        zfs_create(home, options={'mountpoint':'/home', 'compression':'on',
            'devices':'off', 'setuid':'off'})
    progress.advance()

    # Make sure we are creating new boot environment
    for be_no in range(50):
        bootenv = root + '/osdy-' + str(be_no)
        if not zfs_exists(bootenv):
            break
        else:
            bootenv = None

    rootdir = mkdtemp(prefix='install-')
    progress.text = 'Creating ' + bootenv
    zfs_create(bootenv, options={'mountpoint':'legacy'})
    call(['mount', '-F', 'zfs', bootenv, rootdir])
    progress.advance()

    for fs in befs:
        p = bootenv + fs['name']
        progress.text = 'Creating ' + p
        zfs_create(p, options=fs['options'])
        progress.advance()

    # Will rollback if debootstrap fails
    call(['zfs', 'snapshot', '-r', bootenv + '@empty'])

def choose_hdd(number_of_zpools=0):
    hdd_items = map(lambda x: '{}: {} {}'.format(x.name, x.capacity, x.description), hdds)
    choice = None

    buttons = []
    buttons.append(('Use selected disk', 'ok'))
    if number_of_zpools > 0:
        buttons.append(('Back', 'zpool'))
    buttons.append(('Cancel', 'cancel'))


    while not choice in ['ok', 'zpool', 'cancel']:
        (choice, hdd_no) = ListboxChoiceWindow(screen,
                title='Choose hard disk drive',
                text='Choose a disk for the root ZFS pool',
                items=hdd_items,
                buttons=buttons,
                width=76,
                scroll=1,
                default=0,
                help=None
                )
        if choice in ['ok', None]:
            choice = configure_partitions(hdds[hdd_no], len(hdds))
            if choice == 'another':
                choice = None

    if choice == 'ok':
        return hdds[hdd_no]
    if choice == 'zpool':
        return None
    raise Abort('HDD choosing')

def configure_partitions(hdd, number_of_disks=1):
    global solaris_partition
    choice = None
    top_text = 'For the root ZFS pool you need a disk containing a Solaris partition (id 0xbf). \
If this disk includes such a partition and you are happy with it, use this disk. \
Otherwise you have to change partitioning. Note that only the first Solaris partition \
will be used.'

    while not choice in ['ok', 'another', 'cancel']:
        part_info = top_text
        part_info += '\n\n'

        have_partitions = len(hdd.partitions) > 0
        have_gpt = False
        solaris_partition = None
        buttons = []

        if have_partitions:
            part_info += 'Partitions on {name} {desc} {size}:\n\n'.format(
                    name=hdd.name, desc=hdd.description, size=hdd.capacity)
            part_info += '    TYPE    ID    NAME                     SIZE\n'
            n = 0
            for p in hdd.partitions:
                n = n + 1
                if p.id == 0xBF and solaris_partition == None:
                    solaris_partition = p
                if p.id == 0xEE:
                    have_gpt = True
                part_info += '#{n:<2} {primary} {id:#04x} {system:25} {capacity:10}\n'.format(
                    n=n, capacity=p.capacity, system=p.system, id=p.id,
                    primary=['primary', 'logical'][p.logical])
        else:
            part_info += 'This disk is not partitioned.\n'

        if solaris_partition != None:
            buttons.append(('Use this disk', 'ok'))

        if have_gpt:
            part_info += '\nThis disk has GUID Partition Table (GPT) and is unsupported.\n'
            # XXX gparted
        else:
            buttons.append(('Edit partitions', 'fdisk'))

        if number_of_disks > 1:
            buttons.append(('Back', 'another'))

        buttons.append(('Cancel', 'cancel'))

        choice = ButtonChoiceWindow(screen,
                title='Partitions of {}'.format(hdd.name),
                text=part_info,
                buttons=buttons,
                width=76,
                help=None
                )
        if choice == 'fdisk':
            screen.suspend()
            call(['cfdisk', hdd.raw_device])
            hdd.reread_partitions()
            screen.resume()
    # end while

    if choice == 'cancel':
        raise Abort('Partitioning')
    return choice

def get_mirror():
    if not hasattr(get_mirror, "m"):
        get_mirror.m=''
    while True:
        (choice, entry) = EntryWindow(screen,
                title='Enter APT repository URL',
                text='If you want to have line '
                '"deb ftp://192.168.1.1 bok main" in /etc/apt/sources.list '
                'enter "ftp://192.168.1.1" here.\n'
                'Remember to add URL protocol.\n',
                prompts=[('URL', get_mirror.m)],
                buttons = [ ('Ok', 'ok'), ('Back', 'back'), ('Cancel', 'cancel')],
                width=70,
                entryWidth=50,
                )
        if choice == 'back':
            return None
        if choice in ['ok', None]:
            get_mirror.m = entry[0].strip()
            if valid_mirror(get_mirror.m):
                return get_mirror.m
        if choice == 'cancel':
            raise Abort('Entering an APT mirror')

def choose_physlink():
    pass

def configure_network():
    global physlinks
    if not physlinks:
        physlinks = dladm_show_phys()
    if not physlinks:
        choice = ButtonChoiceWindow(screen, title='Network is unreachable',
                text='No network interfaces found on this system. '
                'This mean that public APT repositories cannot be used '
                'to install Dyson.',
                buttons=['Ok'])
        return
    while True:
        if len(physlinks) > 1:
            link = choose_physlink()
        else:
            link = physlinks[0]
        break

def valid_mirror(mirror):
    progress = ProgressMessage(screen, title='Please wait...',
            text='Checking APT mirror: ' + mirror)
    try:
        o = urlopen('{mirror}/dists/{codename}/Release'.format(
            mirror=mirror, codename=codename), timeout=15)
        have_code_name = False
        have_arch = False
        re_arch = re.compile(r'^Architectures:.+\b{archname}\b.*$'.format(archname=archname))
        re_code = re.compile(r'^Codename: +\b{codename}\b.*$'.format(codename=codename))
        for line in o.read(400).split('\n'):
            if re_code.match(line):
                have_code_name = True
            if re_arch.match(line):
                have_arch = True
            if have_code_name and have_arch:
                return True
        raise URLError('Not a valid Dyson APT repository')
    except HTTPError as e:
        ButtonChoiceWindow(screen,
                title='Error',
                text='{code} {reason}.\n\nMirror {mirror} is not usable.'.format(
                    mirror=mirror, code=e.code, reason=e.reason),
                buttons = ['Ok'], width=70)
    except URLError as e:
        ButtonChoiceWindow(screen,
                title='Error',
                text='{reason}.\n\nMirror {mirror} is not usable.'.format(
                    reason=e.reason,mirror=mirror),
                buttons = ['Ok'], width=70)
    except ValueError:
        ButtonChoiceWindow(screen,
                title='Failed to check mirror',
                text='Invalid URL: ' + mirror,
                buttons = ['Ok'], width=70)
    except Exception:
        ButtonChoiceWindow(screen,
                title='Failed to check mirror',
                text='Unknown error: ' + mirror,
                buttons = ['Ok'], width=70)
    return False

def configure_mirror():
    global mirror
    global mirrors
    maxlen = 0
    for m in mirrors:
        if len(m[0]) > maxlen:
            maxlen = len(m[0])
    items = map(lambda x: '{url: <{0}}  - {info}'.format(maxlen, url=x[0], info=x[1]), mirrors)
    items.append(('Enter another  mirror', None))
    mirror = None
    while not mirror:
        (choice, m) = ListboxChoiceWindow(screen,
                title='Choose APT mirror',
                text='Choose APT repository from which Dyson will be installed',
                items=items,
                buttons=[('Ok','ok'), ('Cancel', 'cancel')],
                width=76,
                scroll=1,
                default=0,
                )
        if choice in ['ok', None]:
            if m == None:
                mirror = get_mirror()
            else:
                if valid_mirror(mirrors[m][0]):
                    mirror = mirrors[m][0]
        if choice == 'cancel':
            raise Abort('Choosing APT mirror')

def choose_profile():
    global profile
    global profiles
    maxlen = 0
    for p in profiles:
        if len(p['name']) > maxlen:
            maxlen = len(p['name'])
    items = map(lambda x: '{profile: <{0}} - {desc}'.format(maxlen, profile=x['name'], desc=x['desc']), profiles)
    while not profile:
        (choice, p) = ListboxChoiceWindow(screen,
                title='Install software',
                text='Choose software profile. Additional software will be installed depending on your choice.',
                items=items,
                buttons=[('Ok','ok'), ('Cancel', 'cancel')],
                width=76,
                scroll=1,
                default=1, # default is "basic"
                )
        if choice == 'cancel':
            raise Abort('Choosing software profile')
        if choice in ['ok', None]:
            profile = p


def debootstrap():
    progress = ProgressBar(screen,
            title='Installing base system, please wait...',
            width=70)

    progress.text = ' '

    read, write = os.pipe()

    os.makedirs(rootdir + '/root')
    log = os.open(rootdir + '/root/debootstrap.log', os.O_WRONLY + os.O_CREAT)

    pid = os.fork()
    if pid == 0:
        os.close(read)
        os.dup2(write, 3)
        os.dup2(log, 1)
        os.dup2(log, 2)
        os.close(log)
        os.close(write)
        try:
            os.execl('/usr/sbin/debootstrap', 'debootstrap', '--debian-installer',
    '--exclude=gawk,aptitude,aptitude-common,libboost-iostreams1.48.0,libboost-iostreams1.49.0,libcwidget3',
                    '--include=illumos-grub,illumos-kernel',
                    codename,  rootdir, mirror)
        except EnvironmentError as e:
            sys._exit(e.errno)
    else:
        os.close(write)
        di = os.fdopen(read, 'r', 1)
        info_re = re.compile(' +')
        m = 'START'
        IA = [] # Arguments for IF, cleaned after each IF
        PA = [] # Arguments for PF, cleaned after each PF
        EA = [] # Arguments for EF, cleaned after each EF
        WA = [] # Arguments for WF, cleaned after each WF
        while True:
            line = di.readline()
            if not line:
                break
            try:
                colon = line.find(':')
                if colon == -1:
                    continue
                cmd = line[:colon]
                info = line[colon+1:].strip()
                text = None
                if cmd == 'P': # "P: 23 454 XYZ" or "P: 123 3445"
                    p = info_re.split(info)
                    if len(p) == 3:
                        m = p[2].strip()
                    progress.progress = math.ceil(miles[m][0] + float(p[0])*(miles[m][1] - miles[m][0]) / float(p[1]))
                elif cmd == 'IA':
                    IA.append(info)
                elif cmd == 'PA':
                    PA.append(info)
                elif cmd == 'EA':
                    EA.append(info)
                elif cmd == 'WA':
                    WA.append(info)
                elif cmd == 'IF':
                    text = (info % tuple(IA))
                    IA = []
                elif cmd == 'PF':
                    text = (info % tuple(PA))
                    PA = []
                elif cmd == 'EF':
                    text = (info % tuple(EA))
                    EA = []
                elif cmd == 'WF':
                    text = (info % tuple(WA))
                    WA = []
                if text:
                    progress.text = text
                    os.write(log, text)
                    os.write(log, '\n')
            except:
                pass
        status = os.wait()[1]
        os.close(log)
        return os.WEXITSTATUS(status)

def install_minimal():
    while True:
        configure_mirror()
        code = debootstrap()
        if code == 0:
            return
        while True:
            choice = ButtonChoiceWindow(screen, title='Installation failed',
                    text='Debootstrap failed.',
                    buttons=[('View log', 'log'), ('Try again', 'again'), ('Cancel', 'cancel')],
                    width=40)
            if choice == 'log':
                screen.suspend()
                call(['less', rootdir + '/root/debootstrap.log'])
                screen.resume()
                continue
            if choice == 'again':
                break
            if choice == 'cancel':
                raise Abort('debootstrap failed')
        p = ProgressMessage(screen, title='Please wait', text='Undoing previous try ...')
        umount_in_bootenv() # debootstrap mounts required FS for us
        call(['zfs', 'rollback', '-r', bootenv + '@empty'])


def apt_get_update(log):
    global apt_cache
    progress = ProgressBar(screen, title='Updating APT cache', text='Please wait ...')
    read, write = os.pipe()
    pid = os.fork()
    if pid == 0:
        os.close(read)
        os.dup2(log, 1)
        os.dup2(log, 2)
        try:
            os.execl('/usr/sbin/chroot', 'chroot apt-get update', rootdir,
                     '/usr/bin/apt-get', '-o', 'APT::Status-Fd=' + str(write), 'update')
        except EnvironmentError as e:
            os.write(log, str(e))
            sys._exit(e.errno)
    else: # Parent
        os.close(write)
        pr = os.fdopen(read, 'r', 1)
        while True:
            line = pr.readline()
            if not line:
                break
            try:
                (status, f, percent, text) = line.split(':')
            except:
                os.write(log, 'Installer failed to parse line "{}"'.format(line))
                continue
            progress.progress = float(percent)
            progress.text = text

        status = os.wait()[1]
        if 0 == status:
            progress.text = "Reading files ..."
            apt_cache = apt.Cache(rootdir=rootdir)
        return os.WEXITSTATUS(status)

def apt_get_install(log):
    global profile
    global profiles
    progress = ProgressBar(screen, title='Installing packages', text='Please wait ...')
    packages = [pkg for pkg in profiles[profile]["packages"] if pkg in apt_cache]
    missed = [pkg for pkg in profiles[profile]["packages"] if not pkg in apt_cache]
    for p in  missed:
        os.write(log, "WARNING: missing package `{}'\n".format(p))

    read, write = os.pipe()
    pid = os.fork()
    if pid == 0:
        os.close(read)
        os.dup2(log, 1)
        os.dup2(log, 2)
        try:
            os.execl('/usr/sbin/chroot', 'chroot apt-get install', rootdir,
                     '/usr/bin/apt-get', '-y', '-o', 'APT::Status-Fd=' + str(write),
                     'install', *packages)
        except EnvironmentError as e:
            os.write(log, str(e))
            sys._exit(e.errno)
    else: # Parent
        os.close(write)
        pr = os.fdopen(read, 'r', 1)
        while True:
            line = pr.readline()
            if not line:
                break
            try:
                (status, f, percent, text) = line.split(':')
                percent = float(percent)
            except:
                os.write(log, 'Installer failed to parse line "{}"'.format(line))
                continue
            if status == 'dlstatus':
                percent = percent / 2.0
            elif status == 'pmstatus':
                percent = 50.0 + percent / 2.0
            else:
                continue
            progress.progress = math.ceil(percent)
            progress.text = text

        status = os.wait()[1]
        # apt-get sometimes reports less than 100% at the end
        progress.progress = 100
        return os.WEXITSTATUS(status)


def install_profile():
    global profile
    while True:
        if not profile:
            return
        logfile = rootdir + '/root/dyson-install.log'
        log = os.open(logfile, os.O_WRONLY + os.O_CREAT)
        rc = apt_get_update(log)
        if 0 == rc:
            rc = apt_get_install(log)
        os.close(log)
        if 0 == rc:
            return

        while True:
            choice = ButtonChoiceWindow(screen, title='Installation failed',
                    text='Something went wrong. This is not fatal, because minimal system'
                         ' is installed. You can search log for failures, fix them'
                         ' (if it was, for example, a network failure) and try again.'
                         ' Or give up and stay with minimal system.',
                    buttons=[('View log', 'log'), ('Try again', 'again'), ('Give up', 'skip')],
                    width=50)
            if choice == 'log':
                screen.suspend()
                call(['less', logfile])
                screen.resume()
                continue
            if choice == 'again':
                break
            if choice == 'skip':
                profile = None
                break


def umount_in_bootenv():
    '''unmount all FS mounted in BE on final cleanup.
    These FS also may be left after interruption or debootstrap failure'''
    global bootenv
    global rootdir

    if not bootenv:
        return
    fslist = ['/dev/fd', '/proc', '/devices', '/home']
    for fs in fslist:
        call(['umount', rootdir + fs], stdout=PIPE, stderr=PIPE)

def in_bootenv(cmd):
    chroot = ['chroot', rootdir]
    chroot += cmd
    pobject = Popen(chroot, stderr=PIPE, stdout=PIPE)
    out, err = pobject.communicate()
    return (pobject.returncode, out, err)


def write_vfstab():
    vfstab='''
#device                        device          mount           FS      fsck    mount   mount
#to mount                      to fsck         point           type    pass    at boot options
#
fd                             -               /dev/fd         fd      -       no      -
swap                           -               /tmp            tmpfs   -       yes     -
/dev/zvol/dsk/{rpool}/swap       -               -               swap    -       no      -
'''.format(rpool=rpool)
    try:
        f = open(rootdir + '/etc/vfstab', 'w')
        print(vfstab, file=f)
        f.close()
    except:
        pass

# http://stackoverflow.com/questions/2532053/validate-a-hostname-string
def isValidNodename(hostname):
    if len(hostname) > 63:
        return False
    return re.match(r'^(?!-)[a-z0-9\-]+(?<!-)$', hostname) != None

def configure_nodename():
    nodename = 'frontier'
    while True:
        choice, entry = EntryWindow(screen, title='Set hostname',
                text='Please enter the hostname for this system',
                prompts=[('Hostname', nodename)],
                width=70, entryWidth=63, buttons = ['Ok'])
        nodename = entry[0].strip()
        if isValidNodename(nodename):
            break
        ButtonChoiceWindow(screen, title='Invalid hostname',
                text='The name "{}" is not a valid hostname'.format(nodename),
                buttons=['Ok'])
    try:
        f = open(rootdir + '/etc/nodename', 'w')
        print(nodename, file=f)
        f.close()
        # XXX See http://bugs.debian.org/247734
        f = open(rootdir + '/etc/hosts', 'a')
        print('127.0.0.1\t'+nodename, file=f)
        f.close()
    except:
        pass

def configure_smf():
    ''' Populate SMF repository.
    We could start svccfg and write all commands to its stdin.
    It is fast, but actual execution of import is slow for each manifest.
    So we start svccfg for each manifest to be able to show real progress.
    /proc and /devices must be mounted in chroot env.
    '''
    try:
        manifests = []
        os.chdir(rootdir)
        for root, dirs, files in os.walk('lib/svc/manifest'):
            manifests += [os.path.join('/', root, xml) for xml in files if fnmatch(xml, '*.xml')]
        progress = ProgressBar(screen, title='Configuring SMF repository', width=70,
                              text='Importing SMF manifests ...', top=len(manifests))
        for m in manifests:
            progress.text = 'Importing ' + m
            progress.advance()
            svccfg_cmd = Popen(['/usr/sbin/chroot', rootdir, '/usr/sbin/svccfg', 'import', m],
                               stderr=PIPE, stdout=PIPE, stdin=PIPE,
                               env={'SVCCFG_REPOSITORY':'/etc/svc/repository.db',
                                    'SVCCFG_CHECKHASH':'1'})
            svccfg_cmd.communicate()
            svccfg_cmd.wait()
    except:
        pass



def configure_packages():
    screen.suspend()
    call(['chroot', rootdir, '/usr/sbin/dpkg-reconfigure', 'tzdata', 'locales'], stderr=PIPE)
    screen.resume()

def create_bootarchive():
    progress = ProgressMessage(screen, title='Please wait')
    progress.text = 'Creating boot archive, please wait ...'
    in_bootenv(['/sbin/bootadm', 'update-archive'])

def mount_in_bootenv():
    call(['mount', '-F' , 'lofs', '/devices', rootdir + '/devices'], stdout=PIPE, stderr=PIPE)
    call(['mount', '-F' , 'fd', '-', rootdir + '/dev/fd'], stdout=PIPE, stderr=PIPE)
    call(['mount', '-F' , 'proc', '-', rootdir + '/proc'], stdout=PIPE, stderr=PIPE)

    # /home is rpool/home, we need this to create a regular user:
    call(['mount', '-F' , 'lofs', '/mnt/'+rpool+'/home', rootdir + '/home'], stdout=PIPE, stderr=PIPE)

def set_root_password():
    entry1 = Entry(width=30, password=1, returnExit=0)
    entry2 = Entry(width=30, password=1, returnExit=1)
    passwd = None
    while not passwd:
        choice, entries = EntryWindow(screen, title='Set root password',
                text='Enter the root password. The password is not shown, you have to type it twice.',
                prompts=[('Password', entry1), ('Confirm', entry2)],
                allowCancel=0, width=40, entryWidth=30, buttons=['Ok'])
        if entries[0] == entries[1]:
            passwd = entries[0]
        else:
            ButtonChoiceWindow(screen, title='Error',
                    text='Entries do not match. Try again.', buttons=['Ok'])
    chpasswd_cmd = Popen(['chpasswd', '-R', rootdir], stderr=PIPE, stdout=PIPE, stdin=PIPE)
    chpasswd_cmd.communicate(input='root:'+passwd)

def create_user():
    login    = Entry(width = 30, password = 0, returnExit = 0)
    passwd1  = Entry(width = 30, password = 1, returnExit = 0)
    passwd2  = Entry(width = 30, password = 1, returnExit = 1)
    realname = Entry(width = 30, password = 0, returnExit = 1)
    userid   = Entry(width = 30, password = 0, returnExit = 1)
    while True:
        choice, entries = EntryWindow(screen, title='Create a regular user',
                text='Enter user login and password. '
                     'The password is not shown, you have to type it twice. '
                     'Real name is optional. User ID will be set automatically, if empty.',
                prompts=[
                    ('Login', login),
                    ('Password', passwd1),
                    ('Password again', passwd2),
                    ('Real name', realname),
                    ('User ID', userid),
                ],
                width=50, entryWidth=30, buttons=['Ok', 'Skip'])
        if choice == 'skip':
            break
        if passwd1.value() != passwd2.value():
            ButtonChoiceWindow(screen, title='Error',
                    text='Password entries do not match. Try again.', buttons=['Ok'])
            continue
        if not login.value():
            ButtonChoiceWindow(screen, title='Error',
                    text='User login name cannot be empty.', buttons=['Ok'])
            continue
        # polkit checks if shell is in /etc/shells, /usr/bin/bash is not, thus /bin/bash
        adduser = ['adduser', '--shell', '/bin/bash', '--add_extra_groups', '--disabled-login']
        if userid.value():
            if userid.value().isdigit():
                adduser += ['--uid', userid.value()]
            else:
                ButtonChoiceWindow(screen, title='Error',
                        text='User ID must be a number, not {}'.format(userid.value()), buttons=['Ok'])
                continue
        adduser += ['--gecos', realname.value()]

        adduser.append(login.value())
        ret, stdout, stderr = in_bootenv(adduser)
        if ret == 0:
            chpasswd_cmd = Popen(['chpasswd', '-R', rootdir], stderr=PIPE, stdout=PIPE, stdin=PIPE)
            chpasswd_cmd.communicate(input='{}:{}'.format(login.value(), passwd1.value()))
            call(['usermod', '-R', rootdir, '-a', '-G', 'sudo', login.value()])
            break
        else:
            ButtonChoiceWindow(screen, title='Failed to create user',
                    text=stderr, buttons=['Ok'])
            continue


def configure_bootenv():
    write_vfstab()
    mount_in_bootenv()

    if profile:
        install_profile()

    in_bootenv(['/usr/sbin/devfsadm'])
    open(rootdir + '/reconfigure', 'w').close()

    configure_nodename()
    set_root_password()
    # quick and dirty: if network is configured on livecd, copy config into boot env
    try:
        shutil.copy2('/etc/ipadm/ipadm.conf',
                rootdir+'/etc/ipadm/ipadm.conf')
    except:
        pass

    if profile:
        create_user()
        configure_packages()

    create_bootarchive()
    configure_smf()


def slice4grub(s):
    '''c0t4d0s0 -> a, c0t4d0s3 -> d'''
    try:
        n = s[s.rfind('s')+1:]
        return chr(int(n) + ord('a'))
    except: # random
        return 'a'


def configure_grub():
    global rslice
    # write grub menu for references:
    try:
        menu = open(rootdir + '/boot/grub/menu.lst', 'w')
        print('# This file is just for references. It is not used by GRUB', file=menu)
        print('# Actual menu.lst used by GRUB is {}/boot/grub/menu.lst'.format(rpool), file=menu)
        print('# where {} is the name of root ZFS pool'.format(rpool), file=menu)
        print(grub_top, file=menu)
        print(grub_entry.format(rpool=rpool, bootenv=bootenv, partition=1, slice='a'), file=menu)
        menu.close()
    except:
        pass

    # Zpool was not created during installation, trying to guess slice name:
    if not rslice:
        slices = get_zpool_slice(rpool)
        if len(slices) == 1 and re.match(r'^\w+s\d+$', slices[0]):
            rslice = slices[0]

    items = []
    default = None
    text = ''
    if rslice:
        text += 'Installing GRUB on the master boot sector (MBR) '
        text += 'overrides any boot manager currently installed: '
        text += 'the system will always boot the GRUB in the solaris partition'
        items.append(('Install GRUB to MBR', 'mbr'))
        items.append(('Install GRUB to a partition only', 'partition'))
        items.append(('Only update GRUB menu', 'menu'))
        if rpool in zpools: # installing on existing pool
            default = 'menu'
        else:
            default = 'mbr'
    else:
        text += 'It looks like root ZFS pool "{rpool}" was not created during this installation, '.format(rpool=rpool)
        text += 'so installing GRUB is not supported. '
        text += 'Laterly you can consult {bootenv}/boot/grub/menu.lst for details.'.format(bootenv=bootenv)
        default = 'skip'
    items.append(('Do nothing', 'skip'))
    button, install = ListboxChoiceWindow(screen, title='Configure GRUB',
            text=text, items=items, buttons=['Ok'], width=50, default=default)
    if install == 'skip':
        return

    progress = ProgressMessage(screen, title='Configuring GRUB')
    installgrub_cmd = ['/usr/sbin/installgrub']
    if install == 'mbr':
        progress.text = 'Installing GRUB to the master boot record ...'
        installgrub_cmd += ['-f', '-m']
    elif install == 'menu': # dry run, just to get partition number
        progress.text = 'Updating GRUB menu ...'
        installgrub_cmd += ['-n']
    else:
        progress.text = 'Installing GRUB to partition ...'

    installgrub_cmd += [rootdir+'/boot/grub/stage1', rootdir+'/boot/grub/stage2',
            '/dev/rdsk/' + rslice]
    installgrub = Popen(installgrub_cmd, stdout=PIPE, stderr=PIPE)
    out, err = installgrub.communicate()
    if installgrub.returncode != 0:
        ButtonChoiceWindow(screen, title='Error',
            text='Installing of GRUB failed: '+err, buttons=['Ok'], width=60)

    m = re.search(r'stage1 written to partition (\d)', out)
    if m:
        partition = m.group(1)
    else:
        partition = 0 # Random
    rpool_path = '/mnt/{rpool}/{rpool}'.format(rpool=rpool)
    try:
        if not exists(rpool_path + '/boot'):
            os.mkdir(rpool_path + '/boot')
        if not exists(rpool_path + '/boot/grub'):
            os.mkdir(rpool_path + '/boot/grub')
        if not exists(rpool_path + '/boot/grub/bootsign'):
            os.mkdir(rpool_path + '/boot/grub/bootsign')
        if not exists(rpool_path + '/boot/grub/bootsign/pool_'+rpool):
            open(rpool_path + '/boot/grub/bootsign/pool_'+rpool, 'w').close()
        if not exists(rpool_path + '/boot/grub/menu.lst'):
            menu = open(rpool_path + '/boot/grub/menu.lst', 'w')
            print(grub_top, file=menu)
            menu.close()
        re_bootfs = re.compile(r'^\s*bootfs\s+' + bootenv)
        have_this_bootenv = False
        menu = open(rpool_path + '/boot/grub/menu.lst', 'r+')
        for line in menu:
            if re_bootfs.match(line):
                have_this_bootenv = True
                break
        if not have_this_bootenv:
            print(grub_entry.format(rpool=rpool, bootenv=bootenv,
                partition=partition, slice=slice4grub(rslice)),
                    file=menu)
        menu.close()
    except EnvironmentError as e:
        text = 'Failed to write GRUB configuration: '
        if e.filename:
            text += e.filename + ': '
        text += e.strerror
        ButtonChoiceWindow(screen, title='Error', text=text, buttons=['Ok'], width=60)


def cleanup(destroy_bootenv=False):
    global bootenv
    global rootdir
    progress = ProgressMessage(screen, title='Cleaning up')
    if bootenv:
        umount_in_bootenv()
        call(['umount', rootdir], stdout=PIPE, stderr=PIPE)
        if destroy_bootenv:
            progress.text='Destroying {}, please wait ...'.format(bootenv)
            call(['zfs', 'destroy', '-r',  bootenv], stdout=PIPE, stderr=PIPE)
        else:
            progress.text = 'Adjusting boot environment "{}" ...'.format(bootenv)
            call(['zfs', 'destroy', '-r',  bootenv + '@empty'], stdout=PIPE, stderr=PIPE)
            call(['zfs', 'set', 'canmount=noauto', bootenv], stdout=PIPE, stderr=PIPE)
            call(['zfs', 'set', 'mountpoint=/', bootenv], stdout=PIPE, stderr=PIPE)

        try:
            os.rmdir(rootdir)
        except:
            pass
        bootenv = None

    progress.text = 'Exporting ZFS pools ...'
    for pool in imported_zpools:
        call(['zpool', 'export', '-f',  pool], stdout=PIPE, stderr=PIPE)

def goodbye():
    ButtonChoiceWindow(screen, title='Success',
            text='The Dyson system is successfully installed and configured. '
            'Hopefully, it will boot :-)',
            buttons=['Reboot'])

screen = SnackScreen()

while True:
    try:
        screen.pushHelpLine('F2 - switch to console, F1 - switch back to the installer')
        welcome()
        screen.pushHelpLine(' ')
        find_hdds()
        configure_rpool()
        configure_zfs()
        choose_profile()
        install_minimal()
        configure_bootenv()
        configure_grub()
        goodbye()
        cleanup()
        break

    except Abort as e:
        choice = ButtonChoiceWindow(screen, title='Cancel installation',
                text='Installation is canceled. Would do you like to start it again or reboot?',
                buttons=[('Restart', 'restart'), ('Reboot', 'reboot')], width=50)
        cleanup(destroy_bootenv=True)
        if choice == 'reboot':
            break
    except NoDisks as e:
        ButtonChoiceWindow(screen, title='Error',
                text='No disks found on the system. '
                'Installation of Dyson is not possible.',
                buttons=['Reboot'])
        break
    except:
        ButtonChoiceWindow(screen, title='FATAL ERROR',
                text='The installer has badly failed:\n'+traceback.format_exc(),
                width=70, buttons=['Restart the installer'])
        cleanup(destroy_bootenv=True)

screen.finish()
sys.exit(0)