#!/usr/bin/env bash ############################################################################# ########################################################################### ### Created by A. Danischewski (c) 20150102 v0.03 ### ## Usage: getsf.bsh <URL|"local.html"|"/full/path/to/local.html"> ## Files will be downloaded to the current directory or a newly created ## subdirectory if there are any folders ## ## Some Sourceforge projects don't provide convenient links to download ## their software, no tar balls, no zip files, no svn or git links yet many ## directories and files - downloading each is a lot of clicking and waiting - ## that is time-consuming. That time that can be applied more productively. ## ## With this code you can recursively download entire Sourceforge projects. ## It will dump all files to the directory from where it is run and ## create subdirectories and load them for each folder link. ## ## Warning this program is simple and not very tested - likely to break at any ## point in time since it relies on the current Sourceforge website layout ## but its simple and straightforward enough I thought I should share this ## for the concept of recursion. Your mileage will vary based on SF website ## updates. ## ## If you fire this up one day and its broken, just take the string ## $(lynx -dump -listonly -nonumbers "${1}" | grep files | sed 's#/download##' \ ## | grep http | grep -v timeline | grep -v \? | grep -v 'files/$' | uniq) ## and make that work to provide the URLS in the file list. ## ## Make sure that the http://sourceforge.net/projects URL hasn't changed and ## if need be adjust according to wherever the new pages live. ## ## Then modify this: curl -b cookie_file -o "${DIRNM}.html" -L "$a" ## and make that work and you should have it basically functioning again. ## ## Note: If you provide Sourceforge a useragent browser string (-A) ## this will break - the Sourceforge webserver has useragent logic ## and it doesn't mind (at the time of this posting) providing files ## to the Curl agent. This is a case of - disguise yourself as ## a web browser and the website becomes ~less~ friendly. ## ## This program is free software: you can redistribute it and/or modify ## it under the terms of the GNU General Public License as published by ## the Free Software Foundation, either version 3 of the License, or ## (at your option) any later version. ## ## This program is distributed in the hope that it will be useful, ## but WITHOUT ANY WARRANTY; without even the implied warranty of ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ## GNU General Public License for more details. ## You should have received a copy of the GNU General Public License #### along with this program. If not, see <http://www.gnu.org/licenses/>. ########################################################################### ############################################################################# ### You can uncomment the next line if its broken to better see whats going on #set -x declare -r STARTURL="${1}" declare -r STARTCMD="${STARTCMD:-${0/^./${PWD}}}" declare -r BASEREF='<base href="http://sourceforge.net/projects">' ### To keep the code simple and flexible by allowing Lynx to handle the initial ### input we lose the ability to prefilter to eliminate parent directories - ### curl -> lynx -stdin | sed could avoid this too but this is pretty easy. ### We concatenate the directories we process to a variable and export it ### to all the child processes to check against - that way we don't loop. TMPPARENT="$(echo "${STARTURL##*/projects/}" |sed 's#/#_#g;s/_$//')" if [[ ! -z "${PARENT}" ]] && (($(! grep -Fc $"${TMPPARENT}" <<< "${PARENT}"))); then PARENT="${PARENT}${TMPPARENT}" elif [[ -z "${PARENT}" ]]; then PARENT="${TMPPARENT}" fi export PARENT STARTCMD PWD ### Export PARENT, STARTCMD (in case run w/relative) and ### PWD to be safe or the shell may complain. ### As far as I know Lynx doesn't provide a base url option, so we need to add a base tag to ### avoid relative path links expanding to file:///. ### If we are starting with a file on the filesystem make sure that our base tag is present ### this is the case of when you download an html file then pull from it when on the fs. if [[ -f "${TMPPARENT}.html" ]] && ((! grep -Fc $"$BASEREF" <<< "${TMPPARENT}.html")); then sed -i "7 i $BASEREF" "$STARTURL" fi function usage() { cat <<EOF Description: Clone a SourceForge project This software wants the html of a Sourceforge project *files* url. Not the project homepage, although if you do by accident it will get that and a bunch more stuff that you probably don't need or want. Instead it is designed to start at the Files tab, it can be the first Files page, then you will get the whole project, or you can choose a subfolder to start. Files will be recursively downloaded to the current directory and newly created subdirectories for any SF folders. This software requires Lynx and Curl (sudo apt-get install curl lynx). Usage: ${0##*/} < "http://SF/Files/URL"| "local.html"| "/fullpath/to/local.html" > E.g. ${0##*/} "http://sourceforge.net/projects/files/<..target project..>" EOF exit 1 } [[ ! -z "${1}" ]] && [[ "${1}" =~ ^- ]] || [[ -z "${1}" ]] && usage ### Arg 1 <string> URL|.html file to process. function process_dir() { for a in $(lynx -dump -listonly -nonumbers "${1}" | grep files | \ sed 's#/download##' | grep http | grep -v timeline | grep -v \? | \ grep -v 'files/$' | uniq); do FILENM="${a##*/}"; FILENM="${FILENM%.*}"; if [[ -z "${FILENM}" ]]; then ### Process as a directory ### Remove "http[s]://sourceforge.net/projects/" and under_bar /'s ### for dir name. DIRNM="$(echo "${a##*/projects/}" |sed 's#/#_#g;s/_$//')" if [[ -d "${DIRNM}" ]] || (($(! grep -Fc $"${DIRNM}" <<< "${PARENT}"))); then ### If the directory exists presuming this entry processed, probably the ### starting parent or a parent link in SF to a link traversed if you need ### to restart it or refresh it you should rename the directory and download ### it again or if its really big you can add options to curl to not clobber. echo "Pid:$$ Skipping ${DIRNM} ..." continue; else mkdir "${DIRNM}" cd "${DIRNM}" echo "Pid:$$ Saving ${a} to ${PWD}/${DIRNM}.html ..." curl -b cookie_file -o "${DIRNM}.html" -L "$a" sed -i "7 i $BASEREF" "${DIRNM}.html" ## Inject base tag for Lynx PARENT="${PARENT}${DIRNM}" ### Add newly processed directory to PARENT string. "${STARTCMD}" "${DIRNM}.html" ### Recursively process directories. cd - 1>/dev/null fi else echo "Pid:$$ Saving ${a} to ${PWD}/${a##*/} ..." curl -b cookie_file -o "${a##*/}" -L "$a"; fi done; } process_dir "${STARTURL}" echo "Pid:$$ Done!!" exit 0
Source Code: getsf.bsh MD5 0743699a1350c1c3d86d665bb1d3c75a
(Google Drive incorrectly identifies this as a bin file - its not, its a bash text file)
Also if you copy and paste this text it may not match because of the html double spacing, you can pipe it through sed and the MD5 sum should match: sed "{:a;N;\$!b a};s/[^\n]\n[^\n]//g;s/\n\n/\n/g;s/\(.*\)\(\n.*\)$/\1\n\2/g"
You may want to try this alias out, seems to work on any SF project: E.g. $ gsfn nasm
alias gsfn='_(){ getsf.bsh "http://sourceforge.net/projects/${1}/files/?source=navbar";}; _'
No comments:
Post a Comment