#!/usr/bin/ksh93

########################################################################
#                                                                      #
#               This software is part of the ast package               #
#                 Copyright (c) 2011-2013 Roland Mainz                 #
#                      and is licensed under the                       #
#                 Eclipse Public License, Version 1.0                  #
#                    by AT&T Intellectual Property                     #
#                                                                      #
#                A copy of the License is available at                 #
#          http://www.eclipse.org/org/documents/epl-v10.html           #
#         (with md5 checksum b35adb5213ca9657e911e9befb180842)         #
#                                                                      #
#                                                                      #
#                 Roland Mainz <roland.mainz@nrubsig.org>              #
#                                                                      #
########################################################################

#
# Copyright (c) 2011, 2013, Roland Mainz. All rights reserved.
#

#
# Written by Roland Mainz <roland.mainz@nrubsig.org>
#

# Solaris needs /usr/xpg6/bin:/usr/xpg4/bin because the tools in /usr/bin are not POSIX-conformant
export PATH='/usr/xpg6/bin:/usr/xpg4/bin:/bin:/usr/bin'

# Make sure all math stuff runs in the "C" locale to avoid problems
# with alternative # radix point representations (e.g. ',' instead of
# '.' in de_DE.*-locales). This needs to be set _before_ any
# floating-point constants are defined in this script).
if [[ "${LC_ALL-}" != '' ]] ; then
	export \
		LC_MONETARY="${LC_ALL}" \
		LC_MESSAGES="${LC_ALL}" \
		LC_COLLATE="${LC_ALL}" \
		LC_CTYPE="${LC_ALL}"
		unset LC_ALL
fi
export LC_NUMERIC='C'


function html_entity_to_ascii_string
{
	nameref outbuf=$1
	typeset inbuf="$2"
	integer inbuf_index=0

	typeset entity
	typeset c
	typeset value

	outbuf=''
	
	# Todo: Add more HTML/MathML entities here
	# Note we use a static variable (typeset -S) here to make sure we
	# don't loose the cache data between calls
	typeset -S -A entity_cache=(
		# entity to ascii (fixme: add UTF-8 transliterations)
		["nbsp"]=' '
		["lt"]='<'
		["le"]='<='
		["gt"]='>'
		["ge"]='>='
		["amp"]='&'
		["quot"]='"'
		["apos"]="'"
	)
    
	while c="${inbuf:inbuf_index++:1}" ; [[ "${c}" != '' ]] ; do
		if [[ "${c}" != '&' ]] ; then
			outbuf+="${c}"
			continue
		fi
        
		entity=''
		while c="${inbuf:inbuf_index++:1}" ; [[ "${c}" != '' ]] ; do
			case "${c}" in
				';')
				break
				;;
			~(Eilr)[a-z0-9#])
				entity+="${c}"
				continue
				;;
			*)
#				debugmsg "error &${entity}${c}#"

				outbuf+="${entity}${c}"
				entity=''
				continue 2
				;;
			esac
		done
        
		value=''
		if [[ -v entity_cache["${entity}"] ]] ; then
#			debugmsg "match #${entity}# = #${entity_cache["${entity}"]}#"
			value="${entity_cache["${entity}"]}"
		else
			if [[ "${entity:0:1}" == '#' ]] ; then
				# decimal literal
				value="${ printf "\u[${ printf "%x" "${entity:1:8}" ; }]" ; }"
			elif [[ "${entity:0:7}" == ~(Eilr)[0-9a-f]+ ]] ; then
				# hexadecimal literal
				value="${ printf "\u[${entity:0:7}]" ; }"
			else
				# unknown literal - pass-through
				value="ENT=|${entity}|"
			fi

			entity_cache["${entity}"]="${value}"

#			debugmsg "lookup #${entity}# = #${entity_cache["${entity}"]}#"
		fi

		outbuf+="${value}"
	done

	return 0
}

function urlencode_string
{
	nameref outbuf="$1"
	typeset inbuf="$2"
	typeset c
	typeset buf=''
	integer i
	integer inbuflen=${#inbuf}

	for (( i=0 ; i < inbuflen ; i++ )) ; do
		c="${inbuf:i:1}"
		case "${c}" in
			' ') c='+'   ;;
			'!') c='%21' ;;
			'*') c='%2A' ;;
			"'") c='%27' ;;
			'(') c='%28' ;;
			')') c='%29' ;;
			';') c='%3B' ;;
			':') c='%3A' ;;
			'@') c='%40' ;;
			'&') c='%26' ;;
			'=') c='%3D' ;;
			'+') c='%2B' ;;
			'$') c='%24' ;;
			',') c='%2C' ;;
			'/') c='%2F' ;;
			'?') c='%3F' ;;
			'%') c='%25' ;;
			'#') c='%23' ;;
			'[') c='%5B' ;;
			'\') c='%5C' ;; # we need this to avoid the '\'-quoting hell
			']') c='%5D' ;;
			*)   ;;
		esac
		buf+="${c}"
	done
	
	outbuf="${buf}"
	
	return 0
}

# parse HTTP return code, cookies etc.
function parse_http_response
{
	nameref response="$1"
	typeset h statuscode statusmsg s
	integer i
    
	# we use '\r' as additional IFS to filter the final '\r'
	IFS=$' \t\r' read -r h statuscode statusmsg  # read HTTP/1.[01] <code>
	[[ "${h}" != ~(Eil)HTTP/ ]]           && { print -u2 -f $"%s: HTTP/ header missing\n" "$0" ; return 1 ; }
	[[ "${statuscode}" != ~(Elr)[0-9]+ ]] && { print -u2 -f $"%s: invalid status code\n"  "$0" ; return 1 ; }

	integer response.statuscode="10#${statuscode}"
	typeset response.statusmsg="${statusmsg}"
    
    	typeset -a response.headers
	
	# collect headers
	while IFS='' read -r s ; do
		[[ "${s}" == $'\r' ]] && break

		# strip '\r' at the end
		s="${s/~(Er)$'\r'/}"
		
		response.headers+=( "${s}" )
	done

	for (( i=0 ; i < ${#response.headers[@]} ; i++ )) ; do
		s="${response.headers[i]}"
		# add compound variable fields _ONLY_ on _demand_ if the
		# matching headers exist
		case "${s}" in
			~(Eli)Content-Length:[[:blank:]]+[0-9]+)
				integer response.content_length="10#${s/~(Eli)Content-Length:[[:blank:]]+/}"
				;;
			~(Eli)Content-Type:[[:blank:]]+)
				typeset response.content_type="${s/~(Eli)Content-Type:[[:blank:]]+/}"
				;;
			~(Eli)Location:[[:blank:]]+)
				typeset response.location="${s/~(Eli)Location:[[:blank:]]+/}"
				;;
			~(Eli)Transfer-Encoding:[[:blank:]]+)
				typeset response.transfer_encoding="${s/~(Eli)Transfer-Encoding:[[:blank:]]+/}"
				;;
		esac
	done
	
	return 0
}

function cat_http_body
{
	typeset emode="$1"
	typeset hexchunksize='0'
	integer chunksize=0

	if [[ "${emode}" == 'chunked' ]] ; then
		while IFS=$'\n' read hexchunksize ; do
			hexchunksize="${hexchunksize//$'\r'/}"
			[[ "${hexchunksize}" != '' ]] || continue
			[[ "${hexchunksize}" == ~(Elr)[[:xdigit:]]+ ]] || break
			chunksize="16#${hexchunksize}"
			(( chunksize > 0 )) || break
			dd bs=1 count="${chunksize}" 2>'/dev/null'
		done
	else
		cat
	fi

	return 0
}

function request_google_search
{
	# site setup
	typeset url_host="$1"
	typeset url_path='/search'
	typeset url="http://${url_host}${url_path}"
	integer netfd # http stream number
	typeset input_query="$2"
	integer startpage=$3
	integer numresperpage=$4
	compound httpresponse # http response
	typeset request=''
	integer res

	# we assume "input_query" is a correctly encoded URL which doesn't
	# require any further mangling
	url_path+="?q=${input_query}&start=${startpage}&num=${numresperpage}&filter=0&safe=off"

	request="GET ${url_path} HTTP/1.1\r\n"
	request+="Host: ${url_host}\r\n"
	request+="User-Agent: ${http_user_agent}\r\n"
	request+='Connection: close\r\n'

	redirect {netfd}<> "/dev/tcp/${url_host}/80" 
	(( $? != 0 )) && { print -u2 -f $"%s: Could not open connection to %s.\n" "$0" "${url_host}" ;  return 1 ; }

	# send http get
	{
		print -n -- "${request}\r\n"
	}  >&${netfd}

	# process reply
	parse_http_response httpresponse <&${netfd} ; (( res=$? ))
	if (( res == 0 )) ; then
		response="${ cat_http_body "${httpresponse.transfer_encoding-}" <&${netfd} ; }" ; (( res+=$? ))
	fi
	
	# close connection
	redirect {netfd}<&-
        
	if (( res == 0 && httpresponse.statuscode >= 200 && httpresponse.statuscode <= 299 )) ; then
		print -r -- "${response}"
		return 0
	else
		print -u2 -f $"google response was (%s,%s):\n%s\n" "${httpresponse.statuscode}" "${httpresponse.statusmsg}" "${response}"
		return 1
	fi
	
	# not reached
	return 0
}

function process_ahref
{
	typeset g_provider="$1"
	nameref a=$2
	typeset buf="$3"
	typeset dummy
	integer i	# generic index
	typeset -a ar	# .sh.match data
	typeset c	# used for URLdecoding

	# Notes:
	# 1. (?:<pattern>) is a grouping-only, non-capturing regex expression
	#    (e.g. the string value matching <pattern> does not show up in
	#    ".sh.match")
	# 2. ~(Eix-g) means "extended regular expression ("E"), case-insensitive
	#    ("i"), free-spacing ("x"), non-greedy ("-g")"
	dummy="${buf/~(Eix-g)(?:
		<a
		.+
		href=(?:\"([^\"]*)\"|\'([^\']*)\'|([^[:space:]\"\'][^[:space:]]*)) # capture href link in .sh.match[1]+.sh.match[2]+.sh.match[3]
		(?:[[:space:]].*)? # eat unneeded attribute=value pairs
		>
		(.*) # capture text in .sh.match[5] for usage below
		<\/a>
		)}"

	#
	# postprocessing
	#

	# copy 1D .sh.match that we can use .sh.match in the loop below
	# for other purposes
	for i in "${!.sh.match[@]}" ; do
		ar[i]="${.sh.match[i]}"
	done

	if [[ -v ar[1] || -v ar[2] || -v ar[3] ]] ; then
		typeset href
		html_entity_to_ascii_string href "${ar[1]-}${ar[2]-}${ar[3]-}"

		# add Google base URL if we got a relative URL
		[[ "${href}" == /* ]] && href="http://${g_provider}${href}"

		# if this is a "/url=...sa&" extract the URL part
		# and decode any %XX URLencoded characters
		dummy="${href/~(E)\/url\?q=(.*?)&/dummy}"
		if [[ -v .sh.match[1] ]] ; then
			typeset a.href=''

			href="${.sh.match[1]}"

			for (( i=0 ; i < ${#href} ; i++ )) ; do
				c="${href:i:1}"
				
				# URLdecode
				if [[ "${c}" == '%' ]] ; then
					c="$(printf "\u[${href:i+1:1}${href:i+2:1}]")"
					(( i+=2 ))
				fi
	
				a.href+="$c"
			done
		else
			typeset a.href="${href}"
		fi
	fi

	# s//~(E-g)<.*>/ repeatedly squishes <tag>+</tag>, ~(E-g) does
	# "non-greedy extended regular expression" matching
	[[ -v ar[4] ]] && { typeset a.text ; html_entity_to_ascii_string a.text "${ar[4]//~(E-g)<.*>/}" ; }

	return 0
}

function request_google
{
	typeset str='' next_str=''
	typeset link
	typeset g_provider="$1"
	typeset query="$2"
	compound -a results
	integer i
	integer foundperpage
	integer startpage
	integer -r numresperpage=50
	integer maxresults=$3
	typeset dummy

	urlencode_string query "${query}"

	for (( i=0 , startpage=0 ; i < maxresults ; )) ; do
		next_str="$(
			set -o pipefail
				(
					# make sure we use UTF-8 encoding but preserve LC_ALL's setting
					export LANG='en_US.UTF-8' LC_MESSAGES="${LC_ALL:-${LC_MESSAGES:-${LANG:-"C"}}}"
					unset LC_ALL
					request_google_search "${g_provider}" "${query}" ${startpage} ${numresperpage}
				) | iconv -f 'UTF-8' -
			)"
		if (( $? != 0 )) ; then
			if (( ${#results[@]} == 0 )) ; then
				# no results on the first request ? Return an error then
				return 1
			else
				# stop searching if Google returns an error
				break
			fi
		fi
		
		(( startpage+=numresperpage ))

		(( foundperpage=0 ))	
		while [[ "${next_str}" != '' ]] && (( i < maxresults )) ; do
			str="${next_str}"

			# 1. This is a hack, instead of XML parsing we rely on Google
			# putting the results into "h3" tags with the "class='r'"
			# attribute
			# 2. The right anchor is needed to force non-greedy matching
			# to capture all of the remaining string
			dummy="${str/~(Erix-g)(?:
				<h3
				[[:space:]]+
				class=[\"\']?(?:r|r[[:space:]]+hcw)[\"\']?>)
				(.*) # h3 text stored in .sh.match[1]
				<\/h3>
				(.*) # rest of string, stored in .sh.match[2]
				/}"
			link="${.sh.match[1]-}"
			next_str="${.sh.match[2]-}"
	
			if [[ "${link}" != '' ]] ; then
				process_ahref "${g_provider}" "results[$(( i++ ))]" "${link}"
				(( foundperpage++ ))
			fi
		done
		
		# stop if the results page wasn't full
		(( foundperpage < numresperpage )) && break
	done

	# print results		
	for (( i=0 ; i < ${#results[@]} ; i++ )) ; do
		print -v results[i]
	done

	return 0
}

function usage
{
	OPTIND=0
	getopts -a "${progname}" "${shweblinks_usage}" OPT '-?'
	exit 2
}

function main
{
	typeset service_provider='google.co.uk'
	integer maxnumresults=100

	if [[ -v SHWEBLINKS_SERVICE_PROVIDER ]] ; then
		service_provider="${SHWEBLINKS_SERVICE_PROVIDER}"
	fi
	
	while getopts -a "${progname}" "${shweblinks_usage}" OPT ; do 
		case "${OPT}" in
			'P')	service_provider="${OPTARG}" ;;
			'n')	maxnumresults="${OPTARG}" ;;
			*)	usage ;;
		esac
	done
	shift $(( OPTIND-1 ))
	
	# expecting at least one more argument
	(( $# >= 1 )) || usage
	
	typeset query="$1"
	shift
	
	case "${service_provider}" in
		'google.co.uk')
			request_google 'www.google.co.uk' "${query}" "${maxnumresults}"
			return $?
			;;
		'google.de')
			request_google 'www.google.de' "${query}" "${maxnumresults}"
			return $?
			;;
		*)
			print -u2 -f $"%s: Unsupported service provider.\n" "${progname}"
			return 1
	esac
	
	# not reached
	return 0
}

# program start
builtin basename
builtin cat

set -o noglob
set -o nounset

typeset progname="${ basename "${0}" ; }"

# google only delivers UTF-8 if we pretend to be Mozilla or compatible
typeset -r http_user_agent='Mozilla/5.0 (X11; U; SunOS sun4u; en-US; rv:1.8.1.23) ksh93/shweblinks/2013-05-09'

typeset -r shweblinks_usage=$'+
[-?\n@(#)\$Id: shweblinks (Roland Mainz) 2013-05-09 \$\n]
[-author?Roland Mainz <roland.mainz@nrubsig.org>]
[+NAME?shweblinks - create short list of results from an internet search engine query]
[+DESCRIPTION?\bshweblinks\b is a small utility which passes a given query
	to internet search service and returns a stream of compound variables
	which can be interpreted using a \'read -C var\'-loop.]
[+?The first arg \bquery\b describes query string for an internet search engine.]
[P:provider?Service provider (\'google.co.uk\' or \'google.de\').
	The default can be set via the SHWEBLINKS_SERVICE_PROVIDER environment
	variable.]:[provider]
[n:maxnumresults?Maximum number of results.]:[number]

searchstring

[+SEE ALSO?\bksh93\b(1), \bread\b(1), \bshtinyurl\b(1), http://www.google.co.uk/]
'

main "$@"
exit $?

# EOF.