#!/usr/bin/python3

# -*- coding: utf-8 -*-
# create-upstream-tarballs - Utility to create the required source tarballs
#                            to package Thunderbird within Debian
# Copyright (c) 2022 Carsten Schoenert <c.schoenert@t-online.de>
#
# SPDX-License-Identifier: GPL-2.0-or-later

import argparse
import logging
import os
import requests
import shutil
import sys
import tarfile

from lxml import html
from zipfile import ZipFile

try:
    from pkg_resources import parse_version
except ImportError:
    print('pkg_resources library missing, please install python3-pkg-resources!')
    sys.exit(1)

try:
    from tqdm.auto import tqdm
except ImportError:
    print('tqdm library missing, please install python3-tqdm!')
    sys.exit(1)

# local stuff
import repack


tb_base_url_releases   = 'https://ftp.mozilla.org/pub/thunderbird/releases/'
tb_base_url_candidates = 'https://ftp.mozilla.org/pub/thunderbird/candidates/'

try:
    import colorlog
    HAVE_COLORLOG = True
except ImportError:
    print('colorlog library missing, no colored log possible, please install python3-colorlog if wanted!')
    HAVE_COLORLOG = False


def create_logger():
    '''
    Setup the logging environment.
    '''
    log = logging.getLogger()  # root logger
    log.setLevel(logging.INFO)
    format_str = '%(asctime)s - [%(lineno)4d] %(levelname)-8s- %(message)s'
    date_format = '%Y-%m-%d %H:%M:%S'
    if HAVE_COLORLOG and os.isatty(2):
        cformat = '%(log_color)s' + format_str
        colors = {'DEBUG'   : 'white',
                  'INFO'    : 'green',
                  'WARNING' : 'bold_yellow',
                  'ERROR'   : 'red',
                  'CRITICAL': 'bold_red'}
        formatter = colorlog.ColoredFormatter(cformat,
                                              date_format,
                                              log_colors=colors)
    else:
        formatter = logging.Formatter(format_str, date_format)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)
    log.addHandler(stream_handler)
    return logging.getLogger(__name__)


def option_parser():
    '''
    Creates an argparser for various options.
    '''
    parser = argparse.ArgumentParser(
                description = 'create-tarballs - collect all upstream data and creates '
                              'thunderbird_$version.tar.orig{-l10n}.xz tarballs')

    parser.add_argument('-d', '--debug',
                        action = 'store_true',
                        help   = 'Enable output of debug messages.')

    parser.add_argument('-f', '--filter',
                        action = 'store_true',
                        help   = 'Use dedicated file containing the elements to filter out.')

    parser.add_argument('-g', '--get-versions',
                        action = 'store_true',
                        help   = 'Discover Mozilla CDN and returns the most '
                                 'recent available Thunderbird versions.')

    parser.add_argument('-r', '--repack',
                        action = 'store_true',
                        help   = 'Strip out the unwanted and non needed files and create '
                                 'the thunderbird*.orig.tar.xz tarball, requires optional paramater -v.')

    parser.add_argument('-v', '--version',
                        action = 'store',
                        help   = 'Picks the data for the given version and creates the '
                                 'tarballs for later importing.')

    return parser


def get_website_data(site):
    '''
    Get the raw HTML data from 'site' and returns the HTML dom data.
    '''
    try:
        page_data = requests.get(site)
    except requests.exceptions.ConnectionError as e:
        logging.error('Connection to Mozilla CDN not possible!')
        logging.debug(f'The following error did happen:\n{e}')
        sys.exit(1)

    if page_data.status_code == requests.codes.ok:
        html_tree = html.fromstring(page_data.content)
        return html_tree
    else:
        return 'None'


def get_latest_candidates_build(url):
    '''
    Get the HTML data from the given URL and select all lines that are
    <a href=...> elements and split of the path from href.
    Finally count all found entries so we know the highest available build version.
    '''
    logging.debug(f'Got URL {url}')
    candidates_html = get_website_data(url)
    candidates_href_entries = candidates_html.xpath('//a /@href')
    latest_build = 0
    for elem in candidates_href_entries:
        line = elem.split('/')
        if len(line) == 7:
            if line[5].startswith('build'):
                build_count = int(line[5].strip('build'))
            if build_count > latest_build:
                latest_build = build_count
    logging.debug(f'Found folder \'build{latest_build}/\' as most recent build.')
    return latest_build


def check_for_version_transforming(version):
    '''
    Transform a version string which contains a 'b' into a Debian conform
    string for beta versions.
    '''
    if 'b' in version:
        return '{0}~b{1}'.format(version.split('b')[0],
                                 version.split('b')[1])
    else:
        return version


def get_versions():
    '''
    Checks available Release and Beta versions on the Mozilla CDN site for
    Thunderbird.
    '''

    # Step 1 - Get data about ESR candidates
    #
    # Crawling data containing the final candidates (ESR) versions. New
    # planned ESR versions will be first published and are visible here!
    tb_release_candidates_html = get_website_data(tb_base_url_candidates)

    if tb_release_candidates_html != 'None':
        tb_href_entries = tb_release_candidates_html.xpath('//a /@href')
        # Variable to include all found planned ESR versions.
        tb_upstream_candidates_versions = []
        for elem in tb_href_entries:
            # Processing all href elements and collect all of them with a numeric version.
            line = elem.split('/')
            if (len(line) == 6
                and line[3] == 'candidates'
                and line[4].endswith('candidates')
                and 'b' not in line[4]):
                    # We have now all versions with '-candidates' as suffix which
                    # need to get split of.
                    tb_upstream_candidates_versions.append(line[4].split('-')[0])

    else:
        logging.error(f'Could not load versions from \'{tb_base_url_candidates}\'!')
        sys.exit(1)

    release_planned_version = tb_upstream_candidates_versions[1]
    for pos in range(2, len(tb_upstream_candidates_versions)):
        check = parse_version(release_planned_version) < parse_version(tb_upstream_candidates_versions[pos])
        if check:
            release_planned_version = tb_upstream_candidates_versions[pos]

    # Now searching the newest build for the found candidates version.
    release_candidate_build = get_latest_candidates_build(f'{tb_base_url_candidates}{release_planned_version}-candidates/')
    release_candidate_version = release_planned_version

    # Step 2 - Get data about released ESR and Beta versions
    #
    # Working on data containing the final released (ESR) versions. Found
    # versions here are really released as officially.
    tb_releases_html = get_website_data(tb_base_url_releases)

    if tb_releases_html != 'None':
        tb_href_entries = tb_releases_html.xpath('//a /@href')
        # Variable to include all released ESR versions.
        tb_upstream_versions = []
        for elem in tb_href_entries:
            # Processing all href elements and collect all of them with a numeric version.
            line = elem.split('/')
            if len(line) == 6:
                # Exclude all non numeric elements.
                if (line[4] not in ['custom-updates', 'latest', 'latest-beta']
                    and not line[4].endswith(('esr', '-real'))):
                    tb_upstream_versions.append(line[4])

    else:
        logging.error(f'Could not load versions from \'{tb_base_url_releases}\'!')
        sys.exit(1)

    # Step 3 - Find most recent Release and Beta versions.
    #
    release_version = tb_upstream_versions[1]
    beta_version = release_version
    for pos in range(2, len(tb_upstream_versions)):
        if 'b' not in tb_upstream_versions[pos]:
            check = parse_version(release_version) < parse_version(tb_upstream_versions[pos])
            if check:
                release_version = tb_upstream_versions[pos]

        if 'b' in tb_upstream_versions[pos]:
            check = parse_version(beta_version) < parse_version(tb_upstream_versions[pos])
            if check:
                beta_version = tb_upstream_versions[pos]

    logging.debug(f'Current Release version: {release_version}')
    logging.debug(f'Current (planned) Release version: {release_candidate_version} (build{release_candidate_build})')
    logging.debug(f'Current Beta version: {beta_version}')
    return [release_version, release_candidate_version, release_candidate_build, beta_version]


def get_xpi_languages(base_url):
    '''
    Picks up the available XPI languages from a given URL.
    '''
    logging.debug(f'Using {base_url} to search for xpi files.')
    xpi_html = get_website_data(base_url)
    xpi_href_entries = xpi_html.xpath('//a /@href')
    xpi_list = []
    for elem in xpi_href_entries:
        line = elem.split('/')
        # We'll have a list like this:
        # ['', 'pub', 'thunderbird', 'candidates', '$(version)-candidates', 'build1', 'linux-x86_64', 'xpi', '$(lang).xpi']
        # We can ignore the en-US language, as this is the native language of TB.

        # Catch up XPI data from release channel
        if len(line) == 8 and line[3] == 'releases':
            if not line[7].endswith('en-US.xpi'):
                xpi_list.append(line[7])

        # Catch up XPI data from candidates channel
        if len(line) == 9 and line[3] == 'candidates':
            if not line[8].endswith('en-US.xpi'):
                xpi_list.append(line[8])

    if len(xpi_list) > 0:
        logging.info(f'Found {len(xpi_list)} xpi files on remote side.')
        return xpi_list
    else:
        logging.error('Something went wrong while collecting XPI languages!')
        sys.exit(1)


def select_upstream_version(detected_upstream_versions):
    '''
    Gives back the upstream version the user wants to use.
    '''
    while True:
        print('\nPlease select the Thunderbird version to download!')
        print(f'1 - Current Release version: {detected_upstream_versions[0]}')
        print(f'2 - Current Planned version: {detected_upstream_versions[1]} build{detected_upstream_versions[2]}')
        print(f'3 - Current Beta version   : {detected_upstream_versions[3]}')
        print('4 - Do nothing and Exit\n')
        print('Your selection: ', end=' ')
        version = int(input())
        if version not in [1, 2, 3, 4]:
            print('Wrong input! Please select a valid number for choosing.')
        elif version == 4:
            logging.info('Action aborted by the user.')
            sys.exit(1)
        else:
            print()
            break

    if version == 1:
        return detected_upstream_versions[0]
    elif version == 2:
        return [detected_upstream_versions[1], detected_upstream_versions[2]]
    else:
        return detected_upstream_versions[3]


def download_file(url, file, target_folder):
    '''
    Download a given file.
    If file is already locally available check if a new download is required.
    '''
    with requests.get(url, stream=True) as r:
        # Check header to get content length in bytes.
        total_length = int(r.headers.get('Content-Length'))
        if os.path.isfile(f'{target_folder}/{file}'):
            file_size = os.path.getsize(f'{target_folder}/{file}')
            if total_length != file_size:
                logging.warning(f'Found file \'{file}\' locally, but existing file size differs! Removing!')
                os.unlink(f'{target_folder}/{file}')
            else:
                logging.debug(f'Found file \'{file}\' locally and file size matches upstream size, skipping download.')
                return

        logging.info(f'Download {url}')
        with tqdm.wrapattr(r.raw, 'read', total=total_length, ncols=100, desc='{:9s}'.format(file))as raw:
            with open(f'{target_folder}/{file}', 'wb') as output:
                shutil.copyfileobj(raw, output)


def collect_tb_upstream_data(tb_version):
    '''
    Collect all required upstream files needed to create and recreate the
    tarballs locally.
    '''

    if not isinstance(tb_version, list):
        file = f'thunderbird-{tb_version}.source.tar.xz'
        # Concatenate URLs for ESR or Beta version.
        if 'b' not in tb_version:
            tb_url = f'{tb_base_url_releases}{tb_version}/source/{file}'
            xpi_url = f'{tb_base_url_releases}{tb_version}/linux-x86_64/xpi'
        else:
            build_nb = get_latest_candidates_build(f'{tb_base_url_candidates}{tb_version}-candidates/')
            tb_url = f'{tb_base_url_candidates}{tb_version}-candidates/build{build_nb}/source/{file}'
            xpi_url = f'{tb_base_url_candidates}{tb_version}-candidates/build{build_nb}/linux-x86_64/xpi'
    else:
        file = f'thunderbird-{tb_version[0]}.source.tar.xz'
        # Concatenate URLs for planned ESR version.
        tb_url = f'{tb_base_url_candidates}{tb_version[0]}-candidates/build{tb_version[1]}/source/{file}'
        xpi_url = f'{tb_base_url_candidates}{tb_version[0]}-candidates/build{tb_version[1]}/linux-x86_64/xpi'

    xpi_languages = get_xpi_languages(f'{xpi_url}/')

    if not os.path.isdir(download_folder):
        logging.info(f'Create folder {download_folder}')
        os.mkdir(download_folder)

    # Getting Thunderbird sources.
    download_file(tb_url,
                  file,
                  download_folder)

    # Getting the language XPI files.
    for xpi_file in xpi_languages:
        download_file(f'{xpi_url}/{xpi_file}',
                      xpi_file,
                      download_folder)
    return xpi_languages


def create_tb_l10n_tarball(xpi_languages, base_folder, upstream_version):
    '''
    Create the additional thunderbird_$version.orig-l10n.tar.xz component
    tarball.
    '''
    logging.info('Prepare data for l10n component tarball.')
    l10n_upstream_folder = f'{base_folder}/thunderbird-l10n'
    if os.path.isdir(l10n_upstream_folder):
        if len([entry for entry in os.listdir(l10n_upstream_folder)]) > 0:
            for entry in os.scandir(l10n_upstream_folder):
                if entry.is_dir(follow_symlinks=False) or entry.is_file():
                    if entry.is_dir():
                        logging.debug(f'Removing folder {l10n_upstream_folder}/{entry.name}')
                        shutil.rmtree(f'{l10n_upstream_folder}/{entry.name}')
                    if entry.is_file():
                        logging.debug(f'Removing file {l10n_upstream_folder}/{entry.name}')
                        os.unlink(f'{l10n_upstream_folder}/{entry.name}')
        else:
            logging.debug(f'\'{l10n_upstream_folder}\' exists, but nothing to clean up.')

    else:
        logging.debug(f'Create folder \'{l10n_upstream_folder}\' for xpi l10n upstream data.')
        os.mkdir(l10n_upstream_folder)

    logging.info('Extract l10n data.')
    for xpi in xpi_languages:
        l10n_folder = f'{l10n_upstream_folder}/' + xpi.split('.')[0]
        l10n_xpi_file = f'{base_folder}/{xpi}'
        logging.debug(f'Create folder {l10n_folder}')
        os.mkdir(l10n_folder)
        logging.debug(f'Extract data from {l10n_xpi_file}')
        with ZipFile(l10n_xpi_file, 'r') as zipfile:
            zipfile.extractall(l10n_folder)

    if not isinstance(upstream_version, list):
        # Version number for Beta and ESR versions.
        version = check_for_version_transforming(upstream_version)
    else:
        # Version number for planned ESR version.
        version = f'{upstream_version[0]}'

    l10n_component_name = f'thunderbird_{version}.orig-thunderbird-l10n.tar.xz'
    logging.info(f'Build l10n component tarball {l10n_component_name}')

    # Build the component tarball with the l10n data.
    with tarfile.open(f'{download_folder}/../{l10n_component_name}', "w:xz") as tar:
        tar.add(l10n_upstream_folder, arcname=os.path.basename(l10n_upstream_folder))

    return l10n_component_name


def compare_xpi_languages(l10n_languages_remote):
    '''
    Doings some simple sanity checks to see if a new languages is provided by upstream.
    '''
    # Get listing of the folder thunderbird-l10n.
    l10n_languages_local = os.listdir(f'{os.path.dirname(os.path.abspath(__file__))}/../thunderbird-l10n')
    l10n_languages_local_control = []

    # Get a list l10n packages from debian/control.
    with open(os.path.join(os.path.dirname(__file__), 'control')) as control:
        lines = control.readlines()
        for line in lines:
            if line.startswith('Package: thunderbird-l10n-') and '-all' not in line:
                l10n_languages_local_control.append(line.replace('\n', '').split('Package: thunderbird-l10n-')[1:][0])

    for lang in l10n_languages_remote:
        l10n_language_remote = lang.split('.')[0]

        if l10n_language_remote not in l10n_languages_local:
            logging.warning(f'Found language \'{l10n_language_remote}\' within upstream data, but not in folder thunderbird-l10n/ !')

        if l10n_language_remote.lower() not in l10n_languages_local_control:
            logging.warning(f'Found language \'{l10n_language_remote}\' within upstream data, but not in debian/control !')


def do_repack(source, target, filter_elements):
    '''
    Takes the source tarball and filter out all the given elements into the
    target tarball.
    '''
    logging.debug(f'Create {target} from {source} using {filter_elements}.')
    repack.filter_tar(source, target, filter_elements)


if __name__ == '__main__':
    create_logger()
    logging.info(f'{os.path.basename(sys.argv[0])} started')

    # The base path the tarballs will get placed finally.
    tarball_path = os.path.dirname(os.path.abspath('./'))

    args = option_parser().parse_args()

    if args.debug:
        logging.getLogger().setLevel(logging.DEBUG)
    logging.debug(f'Using args: {args}')

    if not args.filter:
        args.filter = os.path.join(os.path.dirname(__file__), "source.filter")

    if args.repack:
        if args.get_versions:
            logging.error('Option --get-versions can\'t be used together with --repack!')
            option_parser().print_help()
            sys.exit(1)
        if not args.version:
            logging.error('Option --repack requires parameter --version!')
            sys.exit(1)
        else:
            version = args.version
        source = f'{tarball_path}/thunderbird-{version}.source.tar.xz'
        target = f'{tarball_path}/thunderbird_{version}.orig.tar.xz'
        do_repack(source, target, args.filter)

    if args.get_versions:
        upstream_versions = get_versions()
        logging.info(f'Current most recent TB Release ESR version:       {upstream_versions[0]}')
        logging.info(f'Current most recent TB Release candidate version: {upstream_versions[1]} build{upstream_versions[2]}')
        logging.info(f'Current most recent TB Beta version:              {upstream_versions[3]}')
        sys.exit(0)

    if args.version:
        target_version = args.version
        version = args.version
    else:
        target_version = select_upstream_version(get_versions())
        if not isinstance(target_version, list):
            version = check_for_version_transforming(target_version)
        else:
            version = target_version[0]

    download_folder = f'../tb-preparation-{version}'
    logging.debug(f'Upstream version to get: {version}')


    # Download Thunderbird source tarball and also the upstream data of l10n
    # languages Add-ons.
    l10n_languages = collect_tb_upstream_data(target_version)

    # Build the additional Debian packaging component tarball.
    tb_l10n_component_name = create_tb_l10n_tarball(l10n_languages, download_folder, target_version)

    if not isinstance(target_version, list):
        version = check_for_version_transforming(target_version)
        # Catching potential upstream Beta versions without any version
        # string manipulation we need to do for Debian.
        source = f'{download_folder}/thunderbird-{target_version}.source.tar.xz'
    else:
        version = target_version[0]
        source = f'{download_folder}/thunderbird-{version}.source.tar.xz'

    target = f'{tarball_path}/thunderbird_{version}.orig.tar.xz'
    logging.info(f'Build source tarball thunderbird_{version}.orig.tar.xz')
    do_repack(source, target, args.filter)

    logging.info(f'Thunderbird source tarball prepared as:         {tarball_path}/thunderbird_{version}.orig.tar.xz')
    logging.info(f'Thunderbird l10n component tarball prepared as: {tarball_path}/{tb_l10n_component_name}')

    # Some checking if further adjustments are required to have packaging in
    # sync to provided l10n data by upstream.
    compare_xpi_languages(l10n_languages)

# vim:tw=0:
