feat: Refactor download code

This commit is contained in:
Kroese 2024-05-16 21:47:33 +02:00 committed by GitHub
parent 3aee40f160
commit 87e56fe0e7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -69,223 +69,244 @@ handle_curl_error() {
download_windows() { download_windows() {
local HASH="" local HASH=""
local session_id="" local session_id=""
local iso_download_page_html="" local iso_download_page_html=""
local product_edition_id="" local product_edition_id=""
local language_skuid_table_html="" local language_skuid_table_html=""
local sku_id="" local sku_id=""
local iso_download_link_html="" local iso_download_link_html=""
local iso_download_link="" local iso_download_link=""
local windows_version=""
echo "Downloading Windows ${RELEASE} (${I18N})" case "${id,,}" in
# Download newer consumer Windows versions from behind gated Microsoft API "win11${PLATFORM,,}" )
windows_version="11"
;;
"win10${PLATFORM,,}" )
windows_version="10"
;;
"win81${PLATFORM,,}" )
windows_version="8"
;;
* )
echo "Unknown version: $id"
return 1
esac
# Either 8, 10, or 11 echo "Downloading $id"
local windows_version="$1"
local url="https://www.microsoft.com/en-us/software-download/windows$windows_version" local url="https://www.microsoft.com/en-us/software-download/windows$windows_version"
case "$windows_version" in case "$windows_version" in
8 | 10) url="${url}ISO";; 8 | 10) url="${url}ISO";;
esac esac
local user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0" local user_agent="Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0"
# uuidgen: For MacOS (installed by default) and other systems (e.g. with no /proc) that don't have a kernel interface for generating random UUIDs # uuidgen: For MacOS (installed by default) and other systems (e.g. with no /proc) that don't have a kernel interface for generating random UUIDs
session_id="$(cat /proc/sys/kernel/random/uuid 2> /dev/null || uuidgen --random)" session_id="$(cat /proc/sys/kernel/random/uuid 2> /dev/null || uuidgen --random)"
# Get product edition ID for latest release of given Windows version # Get product edition ID for latest release of given Windows version
# Product edition ID: This specifies both the Windows release (e.g. 22H2) and edition ("multi-edition" is default, either Home/Pro/Edu/etc., we select "Pro" in the answer files) in one number # Product edition ID: This specifies both the Windows release (e.g. 22H2) and edition ("multi-edition" is default, either Home/Pro/Edu/etc., we select "Pro" in the answer files) in one number
# This is the *only* request we make that Fido doesn't. Fido manually maintains a list of all the Windows release/edition product edition IDs in its script (see: $WindowsVersions array). This is helpful for downloading older releases (e.g. Windows 10 1909, 21H1, etc.) but we always want to get the newest release which is why we get this value dynamically # This is the *only* request we make that Fido doesn't. Fido manually maintains a list of all the Windows release/edition product edition IDs in its script (see: $WindowsVersions array). This is helpful for downloading older releases (e.g. Windows 10 1909, 21H1, etc.) but we always want to get the newest release which is why we get this value dynamically
# Also, keeping a "$WindowsVersions" array like Fido does would be way too much of a maintenance burden # Also, keeping a "$WindowsVersions" array like Fido does would be way too much of a maintenance burden
# Remove "Accept" header that curl sends by default # Remove "Accept" header that curl sends by default
echo " - Parsing download page: ${url}" echo " - Parsing download page: ${url}"
iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || {
handle_curl_error $? handle_curl_error $?
return $? return $?
} }
echo -n " - Getting Product edition ID: " echo -n " - Getting Product edition ID: "
# tr: Filter for only numerics to prevent HTTP parameter injection # tr: Filter for only numerics to prevent HTTP parameter injection
# head -c was recently added to POSIX: https://austingroupbugs.net/view.php?id=407 # head -c was recently added to POSIX: https://austingroupbugs.net/view.php?id=407
product_edition_id="$(echo "$iso_download_page_html" | grep -Eo '<option value="[0-9]+">Windows' | cut -d '"' -f 2 | head -n 1 | tr -cd '0-9' | head -c 16)" product_edition_id="$(echo "$iso_download_page_html" | grep -Eo '<option value="[0-9]+">Windows' | cut -d '"' -f 2 | head -n 1 | tr -cd '0-9' | head -c 16)"
echo "$product_edition_id" echo "$product_edition_id"
echo " - Permit Session ID: $session_id" echo " - Permit Session ID: $session_id"
# Permit Session ID # Permit Session ID
# "org_id" is always the same value # "org_id" is always the same value
curl --silent --output /dev/null --user-agent "$user_agent" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://vlscppe.microsoft.com/tags?org_id=y6jn8c31&session_id=$session_id" || { curl --silent --output /dev/null --user-agent "$user_agent" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://vlscppe.microsoft.com/tags?org_id=y6jn8c31&session_id=$session_id" || {
# This should only happen if there's been some change to how this API works # This should only happen if there's been some change to how this API works
handle_curl_error $? handle_curl_error $?
return $? return $?
} }
# Extract everything after the last slash # Extract everything after the last slash
local url_segment_parameter="${url##*/}" local url_segment_parameter="${url##*/}"
echo -n " - Getting language SKU ID: " echo -n " - Getting language SKU ID: "
# Get language -> skuID association table # Get language -> skuID association table
# SKU ID: This specifies the language of the ISO. We always use "English (United States)", however, the SKU for this changes with each Windows release # SKU ID: This specifies the language of the ISO. We always use "English (United States)", however, the SKU for this changes with each Windows release
# We must make this request so our next one will be allowed # We must make this request so our next one will be allowed
# --data "" is required otherwise no "Content-Length" header will be sent causing HTTP response "411 Length Required" # --data "" is required otherwise no "Content-Length" header will be sent causing HTTP response "411 Length Required"
language_skuid_table_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --header "Accept:" --max-filesize 10K --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=a8f8f489-4c7f-463a-9ca6-5cff94d8d041&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=getskuinformationbyproductedition&sessionId=$session_id&productEditionId=$product_edition_id&sdVersion=2")" || { language_skuid_table_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --header "Accept:" --max-filesize 10K --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=a8f8f489-4c7f-463a-9ca6-5cff94d8d041&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=getskuinformationbyproductedition&sessionId=$session_id&productEditionId=$product_edition_id&sdVersion=2")" || {
handle_curl_error $? handle_curl_error $?
return $? return $?
} }
# Limit untrusted size for input validation # Limit untrusted size for input validation
language_skuid_table_html="$(echo "$language_skuid_table_html" | head -c 10240)" language_skuid_table_html="$(echo "$language_skuid_table_html" | head -c 10240)"
# tr: Filter for only alphanumerics or "-" to prevent HTTP parameter injection # tr: Filter for only alphanumerics or "-" to prevent HTTP parameter injection
sku_id="$(echo "$language_skuid_table_html" | grep "${I18N}" | sed 's/&quot;//g' | cut -d ',' -f 1 | cut -d ':' -f 2 | tr -cd '[:alnum:]-' | head -c 16)" sku_id="$(echo "$language_skuid_table_html" | grep "${I18N}" | sed 's/&quot;//g' | cut -d ',' -f 1 | cut -d ':' -f 2 | tr -cd '[:alnum:]-' | head -c 16)"
echo "$sku_id" echo "$sku_id"
echo " - Getting ISO download link..." echo " - Getting ISO download link..."
# Get ISO download link # Get ISO download link
# If any request is going to be blocked by Microsoft it's always this last one (the previous requests always seem to succeed) # If any request is going to be blocked by Microsoft it's always this last one (the previous requests always seem to succeed)
# --referer: Required by Microsoft servers to allow request # --referer: Required by Microsoft servers to allow request
iso_download_link_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --referer "$url" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=6e2a1789-ef16-4f27-a296-74ef7ef5d96b&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=GetProductDownloadLinksBySku&sessionId=$session_id&skuId=$sku_id&language=English&sdVersion=2")" iso_download_link_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --referer "$url" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=6e2a1789-ef16-4f27-a296-74ef7ef5d96b&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=GetProductDownloadLinksBySku&sessionId=$session_id&skuId=$sku_id&language=English&sdVersion=2")"
local failed=0 if ! [ "$iso_download_link_html" ]; then
# This should only happen if there's been some change to how this API works
echo " - Microsoft servers gave us an empty response to our request for an automated download."
return 1
fi
if ! [ "$iso_download_link_html" ]; then if echo "$iso_download_link_html" | grep -q "We are unable to complete your request at this time."; then
# This should only happen if there's been some change to how this API works echo " - WARNING! Microsoft blocked the automated download request based on your IP address."
echo " - Microsoft servers gave us an empty response to our request for an automated download." return 1
failed=1 fi
fi
if echo "$iso_download_link_html" | grep -q "We are unable to complete your request at this time."; then # Filter for 64-bit ISO download URL
echo " - WARNING! Microsoft blocked the automated download request based on your IP address." # sed: HTML decode "&" character
failed=1 # tr: Filter for only alphanumerics or punctuation
fi iso_download_link="$(echo "$iso_download_link_html" | grep -o "https://software.download.prss.microsoft.com.*IsoX64" | cut -d '"' -f 1 | sed 's/&amp;/\&/g' | tr -cd '[:alnum:][:punct:]')"
if [ ${failed} -eq 1 ]; then if ! [ "$iso_download_link" ]; then
echo " Manually download the Windows ${windows_version} ISO using a web browser from: ${url}" # This should only happen if there's been some change to the download endpoint web address
echo " Save the downloaded ISO to: $(realpath "${VM_PATH}")" echo " - Microsoft servers gave us no download link to our request for an automated download!"
echo " Update the config file to reference the downloaded ISO: ./${VM_PATH}.conf" return 1
echo " Continuing with the VM creation process..." fi
return 1
fi
# Filter for 64-bit ISO download URL MIDO_URL="$iso_download_link"
# sed: HTML decode "&" character return 0
# tr: Filter for only alphanumerics or punctuation
iso_download_link="$(echo "$iso_download_link_html" | grep -o "https://software.download.prss.microsoft.com.*IsoX64" | cut -d '"' -f 1 | sed 's/&amp;/\&/g' | tr -cd '[:alnum:][:punct:]')"
if ! [ "$iso_download_link" ]; then
# This should only happen if there's been some change to the download endpoint web address
echo " - Microsoft servers gave us no download link to our request for an automated download. Please manually download this ISO in a web browser: $url"
return 1
fi
echo " - URL: ${iso_download_link%%\?*}"
# Download ISO
FILE_NAME="$(echo "$iso_download_link" | cut -d'?' -f1 | cut -d'/' -f5)"
web_get "${iso_download_link}" "${VM_PATH}" "${FILE_NAME}"
} }
download_windows_server() { download_windows_server() {
local iso_download_page_html="" local id="$1"
local windows_version=""
local enterprise_type=""
# Download enterprise evaluation Windows versions case "${id,,}" in
local windows_version="$1" "win11${PLATFORM,,}-enterprise-eval" )
local enterprise_type="$2" windows_version="windows-11-enterprise"
local PRETTY_RELEASE="" enterprise_type="enterprise"
;;
"win10${PLATFORM,,}-enterprise-eval" )
windows_version="windows-10-enterprise"
enterprise_type="enterprise"
;;
"win10${PLATFORM,,}-enterprise-ltsc-eval" )
windows_version="windows-10-enterprise"
enterprise_type="ltsc"
;;
"win2022-eval" )
windows_version="windows-server-2022"
enterprise_type="server"
;;
"win2019-eval" )
windows_version="windows-server-2019"
enterprise_type="server"
;;
"win2016-eval" )
windows_version="windows-server-2016"
enterprise_type="server"
;;
"win2012r2-eval" )
windows_version="windows-server-2012-r2"
enterprise_type="server"
;;
* )
echo "Unknown version: $id"
return 1
esac
local iso_download_page_html=""
case "${RELEASE}" in echo "Downloading $id"
"10-ltsc") PRETTY_RELEASE="10 LTSC";;
"2012-r2") PRETTY_RELEASE="2012 R2";;
*) PRETTY_RELEASE="${RELEASE}";;
esac
echo "Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} (${I18N})" local url="https://www.microsoft.com/en-us/evalcenter/download-$windows_version"
local url="https://www.microsoft.com/en-us/evalcenter/download-$windows_version" echo " - Parsing download page: ${url}"
iso_download_page_html="$(curl --silent --location --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || {
handle_curl_error $?
return $?
}
echo " - Parsing download page: ${url}" if ! [ "$iso_download_page_html" ]; then
iso_download_page_html="$(curl --silent --location --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { # This should only happen if there's been some change to where this download page is located
handle_curl_error $? echo " - Windows server download page gave us an empty response"
return $? return 1
} fi
if ! [ "$iso_download_page_html" ]; then local COUNTRY="US"
# This should only happen if there's been some change to where this download page is located local CULTURE="en-us"
echo " - Windows server download page gave us an empty response"
return 1
fi
local COUNTRY="US" echo " - Getting download link.."
local CULTURE="en-us" iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=${CULTURE}&country=${COUNTRY}")" || {
# This should only happen if there's been some change to the download endpoint web address
echo " - Windows server download page gave us no download link"
return 1
}
echo " - Getting download link.." # Limit untrusted size for input validation
iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=${CULTURE}&country=${COUNTRY}")" || { iso_download_links="$(echo "$iso_download_links" | head -c 1024)"
# This should only happen if there's been some change to the download endpoint web address
echo " - Windows server download page gave us no download link"
return 1
}
# Limit untrusted size for input validation case "$enterprise_type" in
iso_download_links="$(echo "$iso_download_links" | head -c 1024)" # Select x64 download link
"enterprise") iso_download_link=$(echo "$iso_download_links" | head -n 2 | tail -n 1) ;;
# Select x64 LTSC download link
"ltsc") iso_download_link=$(echo "$iso_download_links" | head -n 4 | tail -n 1) ;;
*) iso_download_link="$iso_download_links" ;;
esac
case "$enterprise_type" in # Follow redirect so proceeding log message is useful
# Select x64 download link # This is a request we make this Fido doesn't
"enterprise") iso_download_link=$(echo "$iso_download_links" | head -n 2 | tail -n 1) ;; # We don't need to set "--max-filesize" here because this is a HEAD request and the output is to /dev/null anyway
# Select x64 LTSC download link iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")" || {
"ltsc") iso_download_link=$(echo "$iso_download_links" | head -n 4 | tail -n 1) ;; # This should only happen if the Microsoft servers are down
*) iso_download_link="$iso_download_links" ;; handle_curl_error $?
esac return $?
}
# Follow redirect so proceeding log message is useful # Limit untrusted size for input validation
# This is a request we make this Fido doesn't iso_download_link="$(echo "$iso_download_link" | head -c 1024)"
# We don't need to set "--max-filesize" here because this is a HEAD request and the output is to /dev/null anyway
iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")" || {
# This should only happen if the Microsoft servers are down
handle_curl_error $?
return $?
}
# Limit untrusted size for input validation MIDO_URL="$iso_download_link"
iso_download_link="$(echo "$iso_download_link" | head -c 1024)" return 0
echo " - URL: $iso_download_link"
# Download ISO
FILE_NAME="${iso_download_link##*/}"
web_get "${iso_download_link}" "${VM_PATH}" "${FILE_NAME}"
OS="windows-server"
} }
getWindows() { getWindows() {
local id="$1" local id="$1"
MIDO_URL=""
case "${id,,}" in case "${id,,}" in
"win11${PLATFORM,,}" ) "win81${PLATFORM,,}" | "win10${PLATFORM,,}" | "win11${PLATFORM,,}" )
download_windows "$id" && return 0 download_windows "$id" && return 0
;; ;;
"win11${PLATFORM,,}-enterprise-eval" ) "win11${PLATFORM,,}-enterprise-eval" )
return 1
;;
"win10${PLATFORM,,}" )
download_windows "$id" && return 0
;;
"win10${PLATFORM,,}-enterprise-eval" )
return 1
;;
"win10${PLATFORM,,}-enterprise-ltsc-eval" )
# download_windows_server windows-10-enterprise ltsc
return 1
;;
"win81${PLATFORM,,}" )
download_windows "$id" && return 0
;;
"win81${PLATFORM,,}-enterprise-eval" )
return 1
;;
"win2022-eval" | "win2019-eval" | "win2016-eval" | "win2012r2-eval" | "win2008r2" )
download_windows_server "$id" && return 0 download_windows_server "$id" && return 0
;; ;;
"win10${PLATFORM,,}-enterprise-eval" | "win10${PLATFORM,,}-enterprise-ltsc-eval" )
download_windows_server "$id" && return 0
;;
"win2022-eval" | "win2019-eval" | "win2016-eval" | "win2012r2-eval" )
download_windows_server "$id" && return 0
;;
"win81${PLATFORM,,}-enterprise-eval" )
MIDO_URL="https://download.microsoft.com/download/B/9/9/B999286E-0A47-406D-8B3D-5B5AD7373A4A/9600.17050.WINBLUE_REFRESH.140317-1640_X64FRE_ENTERPRISE_EVAL_EN-US-IR3_CENA_X64FREE_EN-US_DV9.ISO"
return 0
;;
"win2008r2" )
MIDO_URL="https://download.microsoft.com/download/4/1/D/41DEA7E0-B30D-4012-A1E3-F24DC03BA1BB/7601.17514.101119-1850_x64fre_server_eval_en-us-GRMSXEVAL_EN_DVD.iso"
return 0
;;
* )
echo "Unknown version: $id"
esac esac
return 1 return 1