#!/bin/sh
#
# Spk-mirror - Handle SliTaz packages mirrors. Read the README before
# adding or modifying any code in spk!
#
# Copyright (C) SliTaz GNU/Linux - BSD License
# Author: See AUTHORS files
#
. /usr/lib/slitaz/libspk.sh

# gendb a depot for another arch than the host (e.g. the i486 depot on an
# x86_64 build host): --arch= wins over the host /etc/slitaz/slitaz.conf.
# The arch drives the noarch column of packages.info (i486 = 10 fields).
[ -n "$arch" ] && SLITAZ_ARCH="$arch"

# NOTE: I want to use 'extra' mirrors since we can add any extra mirror
# URLs and not only undigest. See also undigest command - Pankso

#
# Functions
#

# Help and usage
usage() {
	name=$(basename $0)
	cat << EOT

$(boldify $(gettext "Usage:")) $name [urls|info|--options]

$(gettext "Handle SliTaz packages mirrors")

$(boldify $(gettext "Commands:"))
  gendb [dir]  $(gettext "Build package lists from .tazpkg (no wok needed)")

$(boldify $(gettext "Options:"))
  --list       $(gettext "List configured mirrors")
  --rm         $(gettext "Remove an extra mirror")
  --gendb      $(gettext "With a dir: build lists, then register it")
  --fresh      $(gettext "gendb: ignore the cache and re-extract everything")
  --arch=      $(gettext "gendb: target arch for a cross-arch depot (default: host)")
  --debug

$(boldify $(gettext "Examples:"))
  $name http://mirror.slitaz.org/
  $name gendb /home/slitaz/packages
  $name /path/to/packages --gendb   $(gettext "(build lists + register)")
  $name /path/to/packages           $(gettext "(register, lists already built)")

EOT
	exit 0
}

# Check if we have packages ID and lists on mirror.
check_mirror() {
	for file in packages.list packages.$SUM packages.info ID
	do
		echo $(gettext "Checking file:") $file
		# busybox wget spells the existence probe "--spider"; the old "-s"
		# is not a valid option in either busybox OR GNU wget ("invalid
		# option -- 's'"), so check_mirror failed on every file and no extra
		# mirror could ever be added. Use busybox explicitly (consistent with
		# the rest of spk; GNU wget in PATH lacks --no-check-certificate).
		if ! busybox wget $WGET_OPT -q --spider -T 6 ${1}${file} 2>/dev/null; then
			newline
			gettext "Unable to reach:"; colorize 31 " $file"
			newline && return 1
		fi
		status
	done
}

# Get extra mirror DB path
extra_db() {
	# Full sanitized path, not just the hostname: two depots on the same
	# host (…/stable/ and …/testing/) must not collide into one db dir.
	db=$(echo ${url#*://} | sed 's|/*$||' | tr '/' '_')
	debug "extra mirror: $url"
	debug "extra DB: $db"
}

# Display info about an extra repo
extra_info() {
	local db=$1
	debug "extra DB: $extradb/$db"
	nb=$(cat $extradb/$db/packages.$SUM 2>/dev/null | wc -l)
	echo "Extra DB  : "$(boldify $db)
	echo "Extra URL : "$(cat $extradb/$db/mirror)
	gettext "Packages  :"; colorize 32 " $nb"
}

# Generate the package DB lists from the .tazpkg files in $1 — the
# consumer-side counterpart to cook-pkgdb (which builds the same files
# from the wok on the build host). No wok needed: receipts are read
# straight from each .tazpkg, so this works on deployed / minimal envs.
# Output is kept byte-compatible with cook-pkgdb, incl. the field-11
# noarch flag on x86_64/arm ("0" = -any.tazpkg, "1" = -$ARCH.tazpkg).
# Also emits descriptions.txt and bundle.tar.lzma so the resulting depot
# is rechargeable by tazpkg (whose recharge fetches only the bundle),
# not just by spk — a dual spk+tazpkg mirror from a wok-less box.
gen_pkgdb() {
	local pkgsdir pkg base sizes noarch rsum t cache fmt md5 prov P cf m \
		reused built newid
	cd "$1" || return 1
	pkgsdir=$(pwd)
	# Refuse to run two gendb on the same depot at once (e.g. a cron rebuild
	# and a manual one): they share .gendb-cache and rewrite the lists in
	# place, so concurrent runs corrupt both. mkdir is atomic; the lock is
	# released on any exit (incl. Ctrl-C) via the trap.
	if ! mkdir "$pkgsdir/.gendb.lock" 2>/dev/null; then
		gettext "ERROR: another gendb is running here"; newline
		gettext "(remove"; echo " $pkgsdir/.gendb.lock if it is stale)"
		return 1
	fi
	trap 'rmdir "'"$pkgsdir"'/.gendb.lock" 2>/dev/null' EXIT
	# INT/TERM must actually STOP gendb: a bare handler releases the lock
	# and the shell resumes the script — a second gendb could then run
	# concurrently while this one keeps rewriting the lists.
	trap 'rmdir "'"$pkgsdir"'/.gendb.lock" 2>/dev/null; exit 1' INT TERM
	newline
	boldify $(gettext "Generating package lists")
	separator

	# Every list is built as <name>.new and renamed into place together at
	# the end, then ID is published LAST: these files are served over http
	# while gendb runs, and truncating/rewriting them in place left clients
	# minutes of partial lists. -maxdepth 1: the receipt indexing below only
	# sees top-level packages, so a subdir .tazpkg must not enter the lists
	# either (it would be listed but unresolvable by full_package).
	gettext "Creating packages.list / packages.md5..."
	find . -maxdepth 1 -name '*.tazpkg' | sed 's|^\./||; s|\.tazpkg$||' \
		| sort > packages.list.new
	find . -maxdepth 1 -name '*.tazpkg' -exec $CHECKSUM '{}' \; \
		| sed 's|\./||' > packages.md5.new
	status

	# packages.info + packages.equiv + files.list — need each receipt.
	# Incremental: each package's contribution is cached keyed by its md5
	# (.gendb-cache/<md5>.i = packages.info line, .f = files.list lines).
	# Re-running gendb after adding/changing a few packages only extracts
	# those — unchanged ones are replayed straight from the cache. The
	# cache is wiped with --fresh or when the format/arch marker changes.
	gettext "Indexing receipts..."; newline
	cache=".gendb-cache"; fmt="1 ${SLITAZ_ARCH}"; reused=0; built=0; skipped=""
	if [ "$fresh" ] || [ "$(cat $cache/.fmt 2>/dev/null)" != "$fmt" ]; then
		rm -rf "$cache"
	fi
	mkdir -p "$cache"; echo "$fmt" > "$cache/.fmt"

	t=$(mktemp -d)
	# Map filename->md5 once, then emit "md5<TAB>pkg" in glob order (the
	# sorted order the old `for pkg in *.tazpkg` loop used). One awk pass
	# instead of an awk-per-package scan of packages.md5 — that lookup
	# alone was ~0.75s on a 600-pkg depot.
	ordered=$(mktemp)
	printf '%s\n' *.tazpkg | awk 'NR==FNR{m[$2]=$1;next} m[$0]!=""{print m[$0]"\t"$0}' \
		packages.md5.new - > "$ordered"

	# Extract cache misses only. On a warm run the test below is a shell
	# builtin (no fork per package), so unchanged packages cost almost
	# nothing; only new/changed ones spawn cpio.
	while IFS='	' read md5 pkg; do
		# Reuse only a COMPLETE cache entry (all three fragments). .i is
		# written last so its presence normally implies .f/.d, but a depot
		# carrying a half-written set from an older gendb still gets
		# re-extracted rather than served a missing files.list.
		if [ -s "$cache/$md5.i" ] && [ -f "$cache/$md5.f" ] && \
			[ -f "$cache/$md5.d" ]; then
			reused=$((reused + 1)); continue
		fi
		rm -rf "$t"/* 2>/dev/null
		( cd "$t" && cpio -i --quiet receipt files.list md5sum \
			description.txt ) < "$pkgsdir/$pkg" 2>/dev/null
		[ -f "$t/receipt" ] || continue
		# The receipt is sourced and every fragment written inside ONE
		# subshell: a broken receipt (a syntax error aborts the shell
		# that sources it) kills the subshell, not gendb — the package
		# is skipped and, having no .i, gets retried on the next run.
		if ! (
		unset PACKAGE VERSION EXTRAVERSION CATEGORY SHORT_DESC WEB_SITE \
			TAGS PACKED_SIZE UNPACKED_SIZE DEPENDS RSUM PROVIDE
		. "$t/receipt" >/dev/null 2>&1 || exit 1
		[ -n "$PACKAGE" ] || exit 1
		sizes=$(echo $PACKED_SIZE $UNPACKED_SIZE | sed 's|\.0||g')
		# RSUM fallback (older receipts lack it) — same recipe as cook-pkgdb.
		rsum="$RSUM"
		if [ -z "$rsum" ] && [ -f "$t/md5sum" ]; then
			{ cat "$t/md5sum"; md5sum "$t/receipt" | sed 's| [^ ]*/| |'
			  [ -f "$t/description.txt" ] && \
				md5sum "$t/description.txt" | sed 's| [^ ]*/| |'
			} > "$t/.rsum"
			rsum=$(md5sum "$t/.rsum" | awk '{print $1}')
		fi
		# Field 11 (noarch flag) only on x86_64/arm; i486 = 10 fields.
		noarch=''
		case "$SLITAZ_ARCH" in
			arm*|x86_64)
				case "$pkg" in
					*-any.tazpkg) noarch='\t0' ;;
					*)            noarch='\t1' ;;
				esac ;;
		esac
		# Write .f and .d FIRST, then .i LAST (atomically). The warm-run reuse
		# test keys on .i alone, so making .i the last fragment written turns
		# it into this entry's commit marker: a gendb interrupted mid-package
		# (Ctrl-C, OOM) leaves no .i, and the package is re-extracted next run
		# instead of being served with a missing/partial files.list or
		# description (the old order wrote .i first, so an interrupt after it
		# stranded a valid-looking .i beside absent .f/.d).
		if [ -f "$t/files.list" ]; then
			sed "s|^|$PACKAGE: |" "$t/files.list" > "$cache/$md5.f"
		else
			: > "$cache/$md5.f"
		fi
		# descriptions.txt fragment — same block layout as cook-pkgdb:
		# a PACKAGE header line, the body with blank lines blanked to a
		# single space, a guaranteed trailing newline, then a blank
		# separator. Empty when the package ships no description.txt.
		if [ -f "$t/description.txt" ]; then
			{ echo "$PACKAGE"
			  sed 's|^$| |' "$t/description.txt"
			  [ -z "$(tail -c1 "$t/description.txt")" ] || echo
			  echo
			} > "$cache/$md5.d"
		else
			: > "$cache/$md5.d"
		fi
		# .i last, via temp+rename so it is all-or-nothing (a truncated .i
		# from an interrupt mid-write would otherwise be reused as valid).
		printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s%b\n' \
			"$PACKAGE" "${VERSION}${EXTRAVERSION}" "$CATEGORY" \
			"$SHORT_DESC" "$WEB_SITE" "$(echo $TAGS)" "$sizes" \
			"$(echo $DEPENDS)" "$rsum" "$(echo $PROVIDE)" "$noarch" \
			> "$cache/$md5.i.tmp" && mv "$cache/$md5.i.tmp" "$cache/$md5.i"
		); then
			rm -f "$cache/$md5.f" "$cache/$md5.d" "$cache/$md5.i"
			colorize 31 "  Skipping broken receipt: $pkg" >&2
			skipped="$skipped $pkg"
			continue
		fi
		built=$((built + 1))
	done < "$ordered"
	rm -rf "$t"

	# Assemble packages.info + files.list from the cache in glob order with
	# one xargs cat each (a couple of execs total, not a cat+cut per
	# package — that loop body was ~4.3s on a 600-pkg depot). Byte-identical
	# to a full rebuild since the order is unchanged.
	awk -v c="$cache" '{print c"/"$1".i"}' "$ordered" | tr '\n' '\0' \
		| xargs -0 cat > packages.info.new 2>/dev/null
	awk -v c="$cache" '{print c"/"$1".f"}' "$ordered" | tr '\n' '\0' \
		| xargs -0 cat > files.list 2>/dev/null
	# descriptions.txt — needed by tazpkg (it goes into bundle.tar.lzma)
	# and by spk-search. Same glob order; empty fragments contribute
	# nothing, matching cook-pkgdb which only emits a block per package
	# that actually ships a description.txt.
	awk -v c="$cache" '{print c"/"$1".d"}' "$ordered" | tr '\n' '\0' \
		| xargs -0 cat > descriptions.txt.new 2>/dev/null
	rm -f "$ordered"

	# packages.equiv in a single awk over packages.info (field 1 = PACKAGE,
	# field 10 = PROVIDE) instead of a grep + in-place sed per PROVIDE
	# (O(n^2), ~2s). Same output: providers prepended (reverse glob order),
	# keys in first-seen order, "alias:provider" kept for name:dest forms.
	awk -F'\t' '
		{
			pkg=$1; np=split($10, pr, " ")
			for(j=1;j<=np;j++){
				p=pr[j]; ci=index(p,":")
				if(ci){ key=substr(p,1,ci-1); ent=substr(p,ci+1)":"pkg }
				else  { key=p; ent=pkg }
				if(key in seen) map[key]=ent" "map[key]
				else { map[key]=ent; seen[key]=1; ord[++k]=key }
			}
		}
		END{ for(i=1;i<=k;i++) print ord[i]"="map[ord[i]] }
	' packages.info.new > packages.equiv.new

	# Prune cache fragments whose package is no longer in the depot.
	for cf in "$cache"/*.i "$cache"/*.f "$cache"/*.d; do
		[ -f "$cf" ] || continue
		m=$(basename "$cf"); m=${m%.*}
		grep -q "^$m  " packages.md5.new || rm -f "$cf"
	done

	# files-list.lzma needs an lzma compressor — optional for spk (only
	# spk-search --file uses it). Skip with a warning on minimal envs.
	if command -v lzma >/dev/null 2>&1; then
		gettext "Compressing files-list..."
		sort -k2 files.list > files.list.sorted
		lzma -c files.list.sorted > files-list.lzma.new
		rm -f files.list.sorted
		$CHECKSUM files-list.lzma.new | cut -d ' ' -f1 | tr -d '\n' \
			> files-list.md5.new
		status
	else
		gettext "No lzma: skipping files-list.lzma + bundle (tazpkg recharge off)"
		newline
	fi
	rm -f files.list

	# A skipped (broken-receipt) package is absent from packages.info: drop
	# it from list/md5 too, or the depot publishes an incoherent catalogue
	# (check_pkgup finds the sum, spk-add can't resolve the package).
	for pkg in $skipped; do
		awk -v f="$pkg" '$NF != f' packages.md5.new > packages.md5.new.$$ \
			&& mv packages.md5.new.$$ packages.md5.new
		grep -vxF "${pkg%.tazpkg}" packages.list.new > packages.list.new.$$
		mv packages.list.new.$$ packages.list.new
	done

	# ID covers the CONTENT of the published lists, not just the file set:
	# a re-index that changes packages.info with the same .tazpkg files
	# (--arch=, fixed gendb, skipped receipt) must yield a new ID, or every
	# client keeps saying "up-to-date" against the old catalogue.
	newid=$(cat packages.md5.new packages.info.new | $CHECKSUM | cut -d' ' -f1)

	# Publish: rename every finished list into place (near-atomic each).
	for f in packages.list packages.md5 packages.info packages.equiv \
		descriptions.txt files-list.lzma files-list.md5; do
		[ -f "$f.new" ] && mv -f "$f.new" "$f"
	done

	# bundle.tar.lzma — what tazpkg's recharge actually downloads (it
	# no longer fetches the loose files; a missing bundle = recharge
	# failure). Built from the just-published lists, then renamed too.
	if command -v lzma >/dev/null 2>&1; then
		gettext "Bundling DB for tazpkg..."
		# Only synthesize empty placeholders when the depot doesn't already
		# ship these lists: a real extra.list from cook-pkgdb was being blanked
		# by `: > extra.list` and then bundled empty, so clients recharged an
		# empty extra list. Remove afterwards only the placeholders we created.
		local made_mirrors=
		[ -f mirrors ]    || { : > mirrors; made_mirrors=yes; }
		# extra.list: create an empty one only if the depot ships none, and
		# leave it on disk afterwards (clients fetch it loose at recharge).
		[ -f extra.list ] || : > extra.list
		busybox tar -chf bundle.tar mirrors extra.list files-list.md5 \
			packages.info descriptions.txt packages.md5 packages.list \
			packages.equiv 2>/dev/null
		lzma -c bundle.tar > bundle.tar.lzma.new && \
			mv -f bundle.tar.lzma.new bundle.tar.lzma
		rm -f bundle.tar
		# mirrors was historically a transient placeholder; only remove the
		# one we synthesized, never a real file already in the depot.
		[ "$made_mirrors" ] && rm -f mirrors
		status
	fi

	# Publish the change marker last: every list above is now in place, so a
	# client that sees this new ID is guaranteed a complete, consistent DB.
	printf '%s\n' "$newid" > ID
	{ printf '%s ' "$newid"; date +%s; } > IDs
	separator
	gettext "Packages indexed:"; colorize 32 " $(grep -c . packages.info)"
	[ "$reused" -gt 0 ] && { gettext " (extracted"; colorize 32 " $built"
		gettext ", reused from cache"; colorize 32 " $reused"; echo ")"; }
	newline
	cd - >/dev/null
}

#
# Commands
#

cmd=$1
[ -d "$cmd" ] && cmd=continue

case "$cmd" in
	*usage|*help) usage ;;
	http://*|ftp://|'continue') continue ;;
	undigest)
		echo "TODO: enable official undigest repo" ;;
	gendb)
		# Build the package lists from a dir of .tazpkg (no wok). Default
		# to the configured main mirror path when no dir is given.
		dir="$2"
		[ "$dir" ] || dir=$(cat $mirrorurl 2>/dev/null)
		case "$dir" in file://*) dir=${dir#file://} ;; esac
		dir=${dir%/}
		if [ ! -d "$dir" ]; then
			gettext "Not a local directory:"; colorize 31 " $dir"
			newline && exit 1
		fi
		if [ -z "$(ls $dir/*.tazpkg 2>/dev/null)" ]; then
			gettext "No packages found in:"; colorize 31 " $dir"
			newline && exit 1
		fi
		gen_pkgdb "$dir"
		# Propagate gen_pkgdb's status so a refused run (lock held) or a
		# failure is visible to scripts/cron, instead of a blanket exit 0.
		exit $? ;;
	""|--list)
		extra=0
		excolor=32
		newline
		boldify "Mirror info"
		separator
		gettext "Main URL :"
		boldify " $(cat $mirrorurl)"
		if [ -f "$mirrors" ]; then
			nb=$(cat $mirrors | wc -l)
			gettext "Mirrors  :"; colorize $excolor " $nb "
		fi
		if [ -d "$extradb" ]; then
			extra=$(ls $extradb | wc -l)
		fi
		[ "$extra" != 0 ] || excolor=31
		gettext "Extra    :"; colorize $excolor " $extra"
		separator
		newline
		if [ "$list" ]; then
			boldify $(gettext "Official mirrors")
			separator
			cat $mirrors
			separator && newline
			if [ "$extra" != "0" ]; then
				boldify $(gettext "Extra mirrors")
				separator
				cat $extradb/*/mirror
				separator && newline
			fi
		fi
		exit 0 ;;
esac

#
# Handle all urls
#

count=0

for url in $@
do
	[ -d "$url" ] && url="dir:$url"
	case "$url" in
		http://*|https://*|ftp://*|local)
			[ "$count" = 0 ] && newline
			count=$(($count + 1))
			url="${url%/}/"
			debug "checking: $url"
			# Remove extra on --rm
			if [ "$rm" ]; then
				extra_db
				if [ -d "$extradb/$db" ]; then
					gettext "Removing extra mirror:"
					echo -n " $(boldify $db)"
					rm -rf $extradb/$db && status
				else
					gettext "Not an extra mirror:"; colorize 31 " $db"
				fi
				newline && continue
			fi
			# Official mirror — exact whole-line match only: a substring
			# hit (or a missing mirrors file erroring out) used to hijack
			# $mirrorurl for a URL that is merely a prefix of a known one.
			if grep -qxF -- "$url" $mirrors 2>/dev/null; then
				gettext "Enabling: official mirror"; newline
				gettext "Architecture:"; echo " $SLITAZ_ARCH"
				# Current mirror layout: <url>/<arch>/<release>/packages/
				# (i486 keeps the historic <url>/packages/<release>/).
				case "$SLITAZ_ARCH" in
					i486) url="${url%/}/packages/${SLITAZ_RELEASE}/" ;;
					*)    url="${url%/}/${SLITAZ_ARCH}/${SLITAZ_RELEASE}/packages/" ;;
				esac
				echo "URL: $(boldify $url)"
				echo "$url" > $mirrorurl
				newline && continue
			fi
			# Extra mirror
			extra_db
			if [ -d "$extradb/$db" ]; then
				gettext "Extra mirror already exists"; newline
				extra_info $db
			else
				boldify $(gettext "Enabling: extra mirror")
				separator
				echo "URL: $(boldify $url)"
				if ! check_mirror $url; then
					continue
				fi
				mkdir -p $extradb/$db
				echo "$url" > $extradb/$db/mirror
				separator
				gettext "New extra mirror is ready to use"; newline
			fi
			newline ;;
		dir:*)
			# Register a local directory as a mirror by symlinking its
			# package lists. The lists come from cook-pkgdb (build host,
			# from the wok) or from 'spk-mirror gendb <dir>' (consumer
			# side, from the .tazpkg) — this just consumes them.
			dir=${url#dir:}
			pkgsdir=${dir%/}
			# Normalize to an absolute path. `spk-mirror ./packages` (or a
			# bare relative name) otherwise wrote "./packages/" into the
			# mirror file and created symlinks with relative targets resolved
			# from $extradb/local (dead links), and download() treats only
			# /*-style paths as local — so every install from the depot ran
			# wget on "./packages/…" and failed.
			case "$pkgsdir" in
				/*) ;;
				*)  pkgsdir=$(cd "$pkgsdir" 2>/dev/null && pwd) || pkgsdir=${dir%/} ;;
			esac
			debug "dir: $dir"
			# --rm: unregister the local depot (the loop above only
			# handled --rm for http/ftp urls, so a directory argument
			# fell through here and got RE-registered instead).
			if [ "$rm" ]; then
				if [ -d "$extradb/local" ]; then
					gettext "Removing extra mirror:"
					echo -n " $(boldify local)"
					rm -rf $extradb/local && status
				else
					gettext "Not an extra mirror:"; colorize 31 " local"
				fi
				newline && continue
			fi
			nb=$(ls $pkgsdir/*.tazpkg 2>/dev/null | wc -l)
			if [ "$nb" = 0 ]; then
				gettext "No packages found in:"; boldify " $pkgsdir"
				newline && continue
			fi
			# --gendb: (re)build the lists from the .tazpkg before registering.
			# A failed gendb (lock held, broken depot) must abort the
			# registration, not link stale lists as freshly generated.
			if [ "$gendb" ]; then
				gen_pkgdb "$pkgsdir" || exit 1
			fi
			if [ ! -f "$pkgsdir/packages.info" ]; then
				gettext "No package lists in:"; boldify " $pkgsdir"
				newline
				gettext "Build them first: spk-mirror gendb $pkgsdir"
				newline && continue
			fi
			newline
			gettext "Directory:"; boldify " $pkgsdir"
			gettext "Packages :"; colorize 32 " $nb"
			newline
			# Enable it as the local extra mirror via symlinks to the
			# cook-pkgdb lists (kept fresh in the chroot on demand).
			boldify $(gettext "Enabling extra local mirror")
			separator
			db=$extradb/local
			debug "database: $db"
			mkdir -p $db && rm -f $db/*
			echo "$pkgsdir/" > $db/mirror
			for list in packages.list packages.$SUM packages.info \
				packages.equiv files-list.lzma files-list.md5 \
				descriptions.txt extra.list ID
			do
				[ -f "$pkgsdir/$list" ] || continue
				echo -n $(gettext "Linking:") $list
				ln -sf $pkgsdir/$list $db/$list
				status
			done
			separator && newline ;;
		--*) continue ;;
		*)
			[ "$count" = 0 ] && newline
			count=$(($count + 1))
			# Should we handle --rm also here ?
			if [ -d "$extradb/$url" ]; then
				extra_info $(basename $extradb/$url)
				newline && continue
			fi
			gettext "Can't handle:"
			colorize 31 " $url"; newline ;;
	esac
done

exit 0
