pkgbuildup

pkgbuildup is a helper tool for AUR package maintainer to automatic update PKGBUILD files, it use two types of files named PKGBUILD.tpl and PKGBUILDUP.

PKGBUILD.tpl is a template file used to generate new PKGBUILD, it support simple template variable syntax which format looks like {% var %}, the variable name is composed by english letters(a-z, A-Z), underline(_) and numbers(0-9).

PKGBUILDUP for pkgbuildup is corresponding to PKGBUILD for makepkg, its a shell script file, too. User could write code to getting the latest package information and store them to a key-value table, the key name is the template variable name in PKGBUILD.tpl, and pkgbuildup will use it to generate a new PKGBUILD file.

This script uses quite a few external programs during its execution. You need to have at least the following installed to function: coreutils, curl, find (findutils), gawk, gettext, grep, sed.

#!/usr/bin/bash

Install

Run command make install or install through AUR which package name is pkgbuildup-git.

Variables and functions for application

Application name and version

appname="pkgbuildup"
version="0.1"

Show help message

show_help() {
    printf "Description:\n"
    printf "    ${appname} is a helper tool for AUR package maintainer to automatic update PKGBUILD files\n"
    printf "\n"
    printf "Usage:\n"
    printf "    ${appname} [-p <file>] [-l <file>] [-h] [-v]\n"
    printf "\n"
    printf "Options:\n"
    printf "    -p <file>\n"
    printf "        Use an alternate update script (instead of '${UPDATE_SCRIPT}')\n"
    printf "    -l, --listvar <file>\n"
    printf "        List variables in a template file\n"
    printf "    -h, --help\n"
    printf "        Prints help message\n"
    printf "    -v, --version\n"
    printf "        Prints version information\n"
    exit 1
}

Show version

show_version() {
    printf "${appname} ${version}\n"
    exit 1
}

Global variables, could be overwrite in PKGBUILDUP

Ignore warning message. Valid: true or false

IGNORE_WARN="false"

Log file name

PKGBUILDUP_LOG="${appname}_result.log"

Command to upload package to AUR. %s will be replaced by the source package file

UPLOAD_CMD="burp -v %s"

File integrity check to use. Valid: md5, sha1, sha256, sha384, sha512

INTEGRITY_CHECK="md5"

Command to get web page content. %s will be replaced by the page url

GET_PAGE_CMD='/usr/bin/curl -sfL --retry 3 --retry-delay 3 %s'

The download utilities that should use to acquire sources Format: protocol::agent

DLAGENTS=('ftp::/usr/bin/curl -fC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u'
    'http::/usr/bin/curl -fLC - --retry 3 --retry-delay 3 -o %o %u'
    'https::/usr/bin/curl -fLC - --retry 3 --retry-delay 3 -o %o %u'
    'rsync::/usr/bin/rsync --no-motd -z %u %o'
    'scp::/usr/bin/scp -C %u %o')

Overwrite existed file when downloading. Valid: true or false

DOWNLOAD_OVERWRITE="false"

Update script name for pkgbuildup, default is PKGUILDUP

UPDATE_SCRIPT='PKGBUILDUP'

Update script name for makepkg, default is PKGUILD

PKGBUILD_SCRIPT='PKGBUILD'

Basic functions

Show message

msg() {
    local mesg=$1; shift
    printf "==> ${mesg}\n" "$@" >&2
}

Show message in level two

msg2() {
    local mesg=$1; shift
    printf "  -> ${mesg}\n" "$@" >&2
}

Show warning message

warning() {
    if [ "${IGNORE_WARN}" = "true" ]; then
        local mesg=$1; shift
        printf "==> $(gettext "WARNING:") ${mesg}\n" "$@" >&2
    fi
}

Show error message

error() {
    local mesg=$1; shift
    printf "==> $(gettext "ERROR:") ${mesg}\n" "$@" >&2
}

Show error message then exit

abort() {
    error "$@"
    error "$(gettext "Aborting...")"
    exit 1
}

Abort if a variable is empty

abort_if_var_empty() {
    local arg=$1
    local argname=$2
    if [[ -z "${arg}" ]]; then
        abort "$(gettext "variable \$%s is empty")" "${argname}"
    fi
}

Warning if a variable is empty

warn_if_var_empty() {
    local arg=$1
    local argname=$2
    if [[ -z "${arg}" ]]; then
        warning "$(gettext "variable \$%s is empty")" "${argname}"
    fi
}

Write message to log file

first_log="true"
log() {
    local mesg=$1; shift
    if [ "${first_log}" = "true" ]; then
        printf "${mesg}\n" "$@" >${PKGBUILDUP_LOG}
        first_log="false"
    else
        printf "${mesg}\n" "$@" >>${PKGBUILDUP_LOG}
    fi
}

Write success message to log file

log_success() {
    log "$(gettext "[SUCCESS]  ")$@"
}

Write failed message to log file

log_failed() {
    log "$(gettext "[FAILED]   ")$@"
}

Source other script file safely

source_safe() {
    shopt -u extglob
    if ! source "$@"; then
        abort "$(gettext "Failed to source %s")" "$1"
    fi
    shopt -s extglob
}

Extract the protocol from a source entry Return local for local sources.

get_protocol() {
    if [[ $1 = *://* ]]; then
        local proto="${1##*::}" # strip leading filename
        printf "%s\n" "${proto%%://*}"
    elif [[ $1 = *lp:* ]]; then
        local proto="${1##*::}"
        printf "%s\n" "${proto%%lp:*}"
    else
        printf "%s\n" local
    fi
}

Get download client

get_downloadclient() {
    local proto=$1

    #- loop through `DOWNLOAD_AGENTS` variable looking for protocol
    local i
    for i in "${DLAGENTS[@]}"; do
        local handler="${i%%::*}"
        if [[ $proto = "$handler" ]]; then
            local agent="${i##*::}"
            break
        fi
    done

    #- if we didn't find an agent, return an error
    if [[ -z $agent ]]; then
        error "$(gettext "Unknown download protocol: %s")" "$proto"
        return 1
    fi

    #- ensure specified program is installed
    local program="${agent%% *}"
    if [[ ! -x $program ]]; then
        local baseprog="${program##*/}"
        error "$(gettext "The download program %s is not installed.")" "$baseprog"
        return 1
    fi

    printf "%s\n" "$agent"
}

Download file Arguments:

  • $1, the file url to download
  • $2 optional, file to save as, default value is the base name of the file url
download_file() {
    local url=$1
    local filename=$2

    #- check arguments
    if [ -z "${filename}" ]; then
        filename="$(basename ${url})"
    fi

    local proto=$(get_protocol "$url")

    #- find the client we should use for this URL
    local dlcmd
    dlcmd=$(get_downloadclient "$proto") || exit $?

    if [[ $proto = "scp" ]]; then
        #- scp downloads should not pass the protocol in the url
        url="${url##*://}"
    fi

    msg2 "$(gettext "Downloading %s...")" "$filename"

    if [ "${DOWNLOAD_OVERWRITE}" != "true" ]; then
        if [ -e "${filename}" ]; then
            msg2 "$(gettext "File existed, skip download")"
            return
        fi
    fi

    #- temporary download file, default to last component of the URL
    local dlfile="${url##*/}"

    #- replace `%o` by the temporary dlfile if it exists
    if [[ $dlcmd = *%o* ]]; then
        dlcmd="${dlcmd//\%o/\"$filename.part\"}"
        dlfile="$filename.part"
    fi
    #- add the URL, either in place of `%u` or at the end
    if [[ $dlcmd = *%u* ]]; then
        dlcmd="${dlcmd//\%u/\"$url\"}"
    else
        dlcmd="$dlcmd \"$url\""
    fi

    local ret=0
    eval "$dlcmd || ret=\$?"
    if (( ret )); then
        [[ ! -s $dlfile ]] && rm -f -- "$dlfile"
        error "$(gettext "Failure while downloading %s")" "$filename"
        return 1
    fi

    #- rename the temporary download file to the final destination
    if [[ $dlfile != "$filename" ]]; then
        mv -f "$dlfile" "$filename"
    fi
}

List variables in template file

Arguments:

  • $1, the template file
listvar() {
    tplfile=$1
    if [ ! -e "${tplfile}" ]; then
        abort "$(gettext "file does not exist, %s")" "${tplfile}"
    fi

    awk '
BEGIN {
    varreg="(^.*){% *(\\w+) *%}(.*$)"
}

varreg {
    while ($0 ~ varreg) {
        var=gensub(varreg, "\\2", "")
        $0=gensub(varreg, "\\1\\3", "")
        if (var in vars == 0) {
            vars[var]=l++
        }
    }
}

END {
    PROCINFO["sorted_in"] = "@val_num_asc"
    for (var in vars) {
        print var
    }
}
' ${tplfile}
}

Join multi-line strings to one line

joinline() {
    echo $*
}

Decode url string, such as convert %2b to +

urldecode() {
    local tmpstr="$(cat - | sed -e 's/%/\\x/g')"
    printf "${tmpstr}\n"
}

Get latest package file name by parsing web page

Used variables:

  • $FILENAME_CMD optional, the command to get latest package file name, if exist %s, will be replaced by $filereg, default value is "grep -o -P \"%s\" | sort -n | tail -1"

Arguments:

  • $1, url of the web page
  • $2, regexp of the file name, used by command $FILENAME_CMD to get the last file name
get_latest_file_by_parsing_web_page() {
    msg2 "$(gettext "Parsing web page to get latest package file name...")"
    local url=$1
    local filereg=$2

    abort_if_var_empty "${url}" "url"
    abort_if_var_empty "${filereg}" "filereg"
    msg2 "$(gettext "  url: %s")" "${url}"
    msg2 "$(gettext "  regexp: %s")" "${filereg}"

    #- get web page content
    local get_page_cmd=$(printf "${GET_PAGE_CMD}" ${url})
    local web_content=$(eval "${get_page_cmd}")

    #- decode url and get latest file
    : ${FILENAME_CMD:="grep -o -P \"%s\" | sort -n | tail -1"}
    filename_cmd="echo '${web_content}' | urldecode | ""$(printf "${FILENAME_CMD}" "${filereg}")"
    local latestfn="$(eval "${filename_cmd}")"

    if [[ -z "${latestfn}" ]]; then
        error "$(gettext "Get latest package file name failed")"
        return 1
    fi

    printf "%s\n" "${latestfn}"
}

Generate PKGBUILD from template file

Arguments:

  • $1, the template file, such as PKGBUILD.tpl

  • $2 optional, the variable values for template file, and echo variable own one line, such as:

    var1=value1
var2=value2
var3=value3

  • $3 optional, the output PKGBUILD file, default is in the same directory with template file

generate_pkgbuild() {
    msg2 "$(gettext "Generating PKGBUILD from template file...")"
    local tplfile=$1
    local varvalues=$2
    local pkgfile=$3

    #- check arguments
    abort_if_var_empty "${tplfile}" "tplfile"
    warn_if_var_empty "${varvalues}" "varvalues"
    if [[ -z "${pkgfile}" ]]; then
        pkgfile="$(dirname ${tplfile})/${PKGBUILD_SCRIPT}"
    fi

    #- replace variables to values in template file to generate PKGBUILD
    #- file, if variable is undefined, just replace to empty
    msg2 "$(gettext "Variable values: %s")" "$(joinline ${varvalues})"
    msg2 "$(gettext "Generate '%s' through template file '%s'")" "${pkgfile}" "${tplfile}"
    awk -v varvalues_str="${varvalues}" '
BEGIN {
    varreg="(^.*){% *(\\w+) *%}(.*$)"

    #- build varvalues array
    split(varvalues_str, lines, "\n")
    for (i in lines) {
        split(lines[i], arr, "=")
        varvalues[arr[1]]=arr[2]
    }
}

varreg {
    tmpstr=$0
    while (tmpstr ~ varreg) {
        var=gensub(varreg, "\\2", "", tmpstr)
        if (var in varvalues) {
            value=varvalues[var]
            $0=gensub("{% *"var" *%}", value, "g")
        }

        #- no matter there are undefined variables, just
        #- remove it in tmpstr to avoid infinity loop
        tmpstr=gensub(varreg, "\\1\\3", "g", tmpstr)
    }
    print
}
' ${tplfile} > ${pkgfile}

    #- check if all the template variables were replaced
    local vars=$(listvar ${pkgfile})
    if [ -n "${vars}" ]; then
        warning "$(gettext "there are undefined variables in %s: %s")" \
            "${pkgfile}" "$(joinline ${vars})"
    fi
}

Generate checksum for file

generate_checksum() {
    local file=$1
    local integ=${INTEGRITY_CHECK}
    msg2 "$(gettext "Generating checksum for file: %s")" ${file}

    if ! type -p openssl >/dev/null; then
        error "$(gettext "Cannot find the command '%s'")" "openssl"
        return 1
    fi

    case "$integ" in
        md5|sha1|sha256|sha384|sha512) : ;;
        *)
            error "$(gettext "Invalid integrity algorithm '%s' specified")" "$integ"
            return 1 ;;
    esac

    local sum="$(openssl dgst -${integ} "${file}")"
    sum=${sum##* }
    printf "%s" "${sum}"
}

Run makepkg

run_makepkg() {
    local pkgdir=$1
    (cd "${pkgdir}"; rm -rf src pkg; makepkg -f) || return 1
}

Upload source package to AUR

upload_package() {
    local pkgdir=$1

    #- generate a new package source
    (cd "${pkgdir}"; makepkg -fS) || return 1;

    local pkgsrc="$(ls --format=single-column --sort=time "${pkgdir}"/*.src.tar.gz | head -1)"
    local upcmd="$(printf ${UPLOAD_CMD} ${pkgsrc})"
    msg2 "$(gettext "Uploading package source to AUR: %s...")" ${pkgsrc}
    eval "${upcmd}" || return 1
}

Helper functions, could use directly

Update PKGBUILD which package file living in other linux distribution's source site, and get the latest package file name by parsing the web page

Used variables:

  • $FILE_INFO, a array with useful information to get the latest package file name, echo element own four fields which were separate by ::

    The element format will looks like as:

    "filename_var::checksum_var::url::filename_regexp"

    Descripion of the format:

    • filename_var, the package file's template variable name
    • checksum_var, the check-sum's template variable name for the package file
    • url, the url of the web page which the package file place in
    • filename_regexp, the regexp to get the latest package file name, will be used by command grep -o -P

    Here is an example:

    FILE_INFO=("filename1::md51::http://ftp.debian.org/debian/pool/main/h/hello::hello_.*?\.tar\.gz"
           "filename2::md52::http://ftp.debian.org/debian/pool/main/h/hello-debhelper::hello-debhelper_.*?\.tar\.gz")

  • $PKGVER_CMD optional, the command to get package version, if exist %s, will be replaced by $pkgverreg, default value is grep -o -P \"%s\" | tail -1

  • $PKGREL_CMD optional, the command to get package release version, , if exist %s, will be replaced by $pkgrelreg, default value is grep -o -P \"%s\" | tail -1

Arguments:

  • $1, the template file
  • $2 optional, package version regexp, used by $PKGVER_CMD in the first package file name, and set the value to template variable pkgver
  • $3 optional, package release regexp, used by $PKGREL_CMD in the first package file name, and set the value to template variable pkgrel
  • $4 optional, if value is true, will run makepkg after updating, default value is true
  • $5 optional, if value is true, will upload package to AUR after upldating, default value is true
do_update_pkgbuild_for_source_site() {
    msg "$(gettext "Update PKGBUILD which package file living in other linux distribution source site...")"
    local tplfile=$1
    local pkgverreg=$2
    local pkgrelreg=$3
    local makepkg="${4:-true}"
    local upload="${5:-true}"
    local varvalues=""          # values for template variable

    abort_if_var_empty "${FILE_INFO[0]}" "FILE_INFO"
    abort_if_var_empty "${tplfile}" "tplfile"
    abort_if_var_empty "${pkgverreg}" "pkgverreg"
    msg2 "$(gettext "Updating %s...")" "${tplfile}"

    #- parse variable FILE_INFO to get each package file's latest name and check-sum
    local mainfn=""
    for info in "${FILE_INFO[@]}"; do
        local filename_var="$(echo $info | awk -F '::' '{print $1}')"
        local checksum_var="$(echo $info | awk -F '::' '{print $2}')"
        local url="$(echo $info | awk -F '::' '{print $3}')"
        local filereg="$(echo $info | awk -F '::' '{print $4}')"

        local latestfn
        latestfn=$(get_latest_file_by_parsing_web_page "${url}" "${filereg}") || return 1
        varvalues=$(printf "%s\n%s" "${varvalues}" "${filename_var}=${latestfn}")

        #- download latest package file
        local fileurl="${url}/${latestfn}"
        local savefile="$(dirname ${tplfile})/${latestfn}"
        download_file ${fileurl} ${savefile} || return 1

        #- generate integrity checks
        local checksum
        checksum=$(generate_checksum ${savefile}) || return 1
        varvalues=$(printf "%s\n%s" "${varvalues}" "${checksum_var}=${checksum}")

        #- set first package as the main package, and mark its latest
        #- file name to get package version later
        if [ -z "${mainfn}" ]; then
            mainfn=${latestfn}
        fi
    done

    #- get package version
    : ${PKGVER_CMD:="grep -o -P \"%s\" | tail -1"}
    pkgver_cmd="echo ${mainfn} | ""$(printf "${PKGVER_CMD}" "${pkgverreg}")"
    msg2 "$(gettext "Getting package version, regexp: %s")" "${pkgverreg}"
    msg2 "$(gettext "Command: %s")" "${pkgver_cmd}"
    local pkgver="$(eval "${pkgver_cmd}")"
    varvalues="$(printf "%s\n%s" "${varvalues}" "pkgver=${pkgver}")"

    #- get package release
    : ${PKGREL_CMD:="grep -o -P \"%s\" | tail -1"}
    pkgrel_cmd="echo ${mainfn} | ""$(printf "${PKGREL_CMD}" "${pkgrelreg}")"
    msg2 "$(gettext "Getting package release, regexp: %s")" "${pkgrelreg}"
    msg2 "$(gettext "Command: %s")" "${pkgrel_cmd}"
    local pkgrel="$(eval "${pkgrel_cmd}")"
    varvalues="$(printf "%s\n%s" "${varvalues}" "pkgrel=${pkgrel}")"

    #- generate new PKGBUILD
    generate_pkgbuild "${tplfile}" "${varvalues}"

    #- run makepkg
    local pkgdir="$(dirname ${tplfile})"
    if [ "${makepkg}" = "true" ]; then
        run_makepkg ${pkgdir} || return 1
    fi

    #- upload package
    if [ "${upload}" = "true" ]; then
        upload_package ${pkgdir} || return 1
    fi
}

Wrapper for do_update_pkgbuild_for_source_site()

update_pkgbuild_for_source_site() {
    local tplfile=$1
    do_update_pkgbuild_for_source_site "$@"
    if (( $? )); then
        log_failed "${tplfile}"
    else
        log_success "${tplfile}"
    fi
}

Main loop

Setup bash

shopt -s extglob

Dispatch arguments

arg_update_script="${PWD}/${UPDATE_SCRIPT}"
arg_listvar=""
arg_help=""
arg_version=""

while [ $# -gt 0 ]; do
    case $1 in
        -p)             arg_update_script="$2"; shift; break ;;
        -l|--listvar)   arg_listvar="$2"; shift; break ;;
        -h|--help)      arg_help=t; break ;;
        -v|--version)   arg_version=t; break ;;
        --)             shift; break ;;
        *)              abort "$(gettext "Unknown argument: %s")" "$@" ;;
    esac
done

Show help message

if [ "${arg_help}" ]; then
    show_help
fi

Show version

if [ "${arg_version}" ]; then
    show_version
fi

List variables in template file

if [ "${arg_listvar}" ]; then
    listvar ${arg_listvar}
    exit
fi

Load update script, default name is PKGBUILDUP

source_safe "${arg_update_script}"

License

GNU General Public License, Version 3.0