xixu-me
ebc102fd60
Replaced the standalone 'purge-jsdelivr-cache' job with an inline step in the main workflow. The new implementation purges jsDelivr CDN cache for PNG files in the 'favicons' directory, improving progress tracking and simplifying the workflow structure.
289 lines
11 KiB
YAML
289 lines
11 KiB
YAML
name: The Workflow
|
|
|
|
on:
|
|
# Weekly trigger every Sunday at 02:00 UTC
|
|
schedule:
|
|
- cron: "0 2 * * 0"
|
|
# Manual trigger with configurable domain count
|
|
workflow_dispatch:
|
|
inputs:
|
|
domain_count:
|
|
description: "Number of top domains to process (supported values: 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000)"
|
|
required: false
|
|
default: "200000"
|
|
type: string
|
|
|
|
env:
|
|
# Default domain count (can be overridden by workflow input)
|
|
DOMAIN_COUNT: ${{ github.event.inputs.domain_count || '200000' }}
|
|
|
|
# Global permissions for the workflow
|
|
permissions:
|
|
contents: write
|
|
actions: write
|
|
|
|
jobs:
|
|
validate-input:
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- name: Validate domain count input
|
|
run: |
|
|
VALID_COUNTS="200 500 1000 2000 5000 10000 20000 50000 100000 200000"
|
|
if [[ ! " $VALID_COUNTS " =~ " ${{ env.DOMAIN_COUNT }} " ]]; then
|
|
echo "Error: Invalid domain count '${{ env.DOMAIN_COUNT }}'. Supported values: $VALID_COUNTS"
|
|
exit 1
|
|
fi
|
|
echo "Domain count validation passed: ${{ env.DOMAIN_COUNT }}"
|
|
|
|
fetch-domain-list:
|
|
runs-on: ubuntu-latest
|
|
needs: validate-input
|
|
permissions:
|
|
contents: read
|
|
actions: write
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Fetch top domains from Cloudflare Radar API
|
|
run: |
|
|
echo "Fetching top ${{ env.DOMAIN_COUNT }} domains from Cloudflare Radar API..."
|
|
curl "https://api.cloudflare.com/client/v4/radar/datasets/ranking_top_${{ env.DOMAIN_COUNT }}" \
|
|
--header "Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}" \
|
|
--output domains_raw.txt
|
|
|
|
- name: Process domain list
|
|
run: |
|
|
echo "Processing domain list - removing first line..."
|
|
# Remove the first line and save to domains file
|
|
tail -n +2 domains_raw.txt > domains
|
|
echo "Domain list processed. Total domains: $(wc -l < domains)"
|
|
|
|
- name: Upload domain list as artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: domain-list-artifact
|
|
path: domains
|
|
retention-days: 7
|
|
|
|
process-favicons-and-release:
|
|
runs-on: ubuntu-latest
|
|
needs: fetch-domain-list
|
|
permissions:
|
|
contents: write
|
|
actions: read
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Download domain list artifact
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: domain-list-artifact
|
|
path: .
|
|
|
|
- name: Create favicons directory
|
|
run: |
|
|
mkdir -p favicons
|
|
echo "Created favicons directory"
|
|
|
|
- name: Download favicons for each domain
|
|
run: |
|
|
echo "Starting favicon download process..."
|
|
echo "Processing top ${{ env.DOMAIN_COUNT }} domains..."
|
|
total_domains=$(wc -l < domains)
|
|
echo "Total domains to process: $total_domains"
|
|
|
|
# Counter for progress tracking
|
|
counter=0
|
|
success_count=0
|
|
error_count=0
|
|
|
|
# Array to track used filenames for duplicate handling
|
|
declare -A filename_count
|
|
|
|
# Read domains file line by line
|
|
while IFS= read -r domain; do
|
|
counter=$((counter + 1))
|
|
|
|
# Skip empty lines
|
|
if [ -z "$domain" ]; then
|
|
continue
|
|
fi
|
|
|
|
# Progress indicator every 1000 domains
|
|
if [ $((counter % 1000)) -eq 0 ]; then
|
|
echo "Progress: $counter/$total_domains domains processed (Success: $success_count, Errors: $error_count)"
|
|
fi
|
|
|
|
# Extract second-level domain (remove top-level domain)
|
|
# For example: google.com -> google, github.io -> github
|
|
second_level_domain=$(echo "$domain" | sed 's/\.[^.]*$//')
|
|
|
|
# Handle duplicate filenames by adding sequential numbers
|
|
base_filename="$second_level_domain"
|
|
if [[ -n "${filename_count[$base_filename]}" ]]; then
|
|
# Increment count for this filename
|
|
filename_count[$base_filename]=$((filename_count[$base_filename] + 1))
|
|
final_filename="${base_filename}_${filename_count[$base_filename]}"
|
|
else
|
|
# First occurrence of this filename
|
|
filename_count[$base_filename]=0
|
|
final_filename="$base_filename"
|
|
fi
|
|
|
|
# Construct favicon URL
|
|
favicon_url="https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://${domain}&size=256"
|
|
favicon_file="favicons/${final_filename}.png"
|
|
|
|
# Download favicon with enhanced error handling and timeout
|
|
if timeout 15 curl -L -s -o "$favicon_file" "$favicon_url" --max-time 10 --retry 3 --retry-delay 1; then
|
|
# Check if file was actually downloaded and has content
|
|
if [ -s "$favicon_file" ]; then
|
|
# Calculate SHA-256 hash of the downloaded file
|
|
file_hash=$(sha256sum "$favicon_file" | cut -d' ' -f1)
|
|
|
|
# Check if the hash matches the failure hash
|
|
if [ "$file_hash" = "59bfe9bc385ad69f50793ce4a53397316d7a875a7148a63c16df9b674c6cda64" ]; then
|
|
# This is a failed download (generic/default favicon)
|
|
rm -f "$favicon_file"
|
|
error_count=$((error_count + 1))
|
|
echo "Warning: Generic favicon detected for domain: $domain (mapped to $final_filename) (SHA-256: $file_hash)"
|
|
else
|
|
# Valid favicon downloaded
|
|
success_count=$((success_count + 1))
|
|
echo "Success: Downloaded favicon for $domain -> $final_filename.png"
|
|
fi
|
|
else
|
|
# Remove empty file
|
|
rm -f "$favicon_file"
|
|
error_count=$((error_count + 1))
|
|
echo "Warning: Empty file for domain: $domain (mapped to $final_filename)"
|
|
fi
|
|
else
|
|
# Remove any partial file
|
|
rm -f "$favicon_file"
|
|
error_count=$((error_count + 1))
|
|
echo "Error: Failed to download favicon for domain: $domain (mapped to $final_filename)"
|
|
fi
|
|
|
|
# Add high failure rate detection (stop if >80% failures after 100 domains)
|
|
if [ $counter -gt 100 ] && [ $((error_count * 100 / counter)) -gt 80 ]; then
|
|
echo "Error: High failure rate detected (${error_count}/${counter}). Stopping execution."
|
|
exit 1
|
|
fi
|
|
done < domains
|
|
|
|
echo "Favicon download completed!"
|
|
echo "Total processed: $counter"
|
|
echo "Successful downloads: $success_count"
|
|
echo "Failed downloads: $error_count"
|
|
echo "Success rate: $((success_count * 100 / counter))%"
|
|
echo "Downloaded favicons saved in: $(ls favicons | wc -l) files"
|
|
|
|
- name: Push favicons to branch
|
|
run: |
|
|
# Configure git
|
|
git config --global user.name 'github-actions[bot]'
|
|
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
|
|
|
# Create or switch to assets branch
|
|
git checkout --orphan assets
|
|
git rm -rf . 2>/dev/null || true
|
|
|
|
# Copy favicons directory contents
|
|
cp -r favicons/* . 2>/dev/null || echo "No favicons to copy"
|
|
|
|
# Add all favicon files
|
|
git add *.png 2>/dev/null || echo "No PNG files to add"
|
|
|
|
# Commit with timestamp
|
|
if git diff --staged --quiet; then
|
|
echo "No favicon changes to commit"
|
|
else
|
|
git commit -m "Update favicons (top ${{ env.DOMAIN_COUNT }}) - $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
|
git push origin assets --force
|
|
fi
|
|
|
|
- name: Purge jsDelivr CDN cache
|
|
run: |
|
|
echo "Purging jsDelivr CDN cache for assets branch..."
|
|
# Change to directory with PNG files
|
|
cd favicons
|
|
|
|
# Count files for progress tracking
|
|
total_files=$(find . -name "*.png" | wc -l)
|
|
echo "Purging cache for $total_files files"
|
|
|
|
count=0
|
|
for file in $(find . -name "*.png" -type f -printf "%f\n"); do
|
|
count=$((count + 1))
|
|
if [ $((count % 100)) -eq 0 ]; then
|
|
echo "Progress: Purged $count/$total_files files"
|
|
fi
|
|
|
|
curl -s -o /dev/null -w "%{http_code}" "https://purge.jsdelivr.net/gh/${{ github.repository }}@assets/${file}"
|
|
done
|
|
|
|
echo "jsDelivr CDN cache purge completed for $count files"
|
|
|
|
- name: Install tzst
|
|
run: |
|
|
echo "Installing tzst..."
|
|
pip install tzst
|
|
tzst --version
|
|
|
|
- name: Create compressed archive
|
|
run: |
|
|
echo "Creating compressed favicon archive..."
|
|
# Create archive with tzst
|
|
tzst a archive.tzst favicons
|
|
|
|
# Verify archive was created
|
|
if [ -f "archive.tzst" ]; then
|
|
echo "Archive created successfully"
|
|
ls -lh archive.tzst
|
|
else
|
|
echo "Error: Archive creation failed"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Generate release tag
|
|
id: tag
|
|
run: |
|
|
# Generate tag based on current date and domain count
|
|
TAG="$(date -u '+%Y-%m-%d')-${{ env.DOMAIN_COUNT }}"
|
|
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
|
echo "Generated release tag: $TAG"
|
|
|
|
- name: Create GitHub Release
|
|
uses: softprops/action-gh-release@v1
|
|
with:
|
|
tag_name: ${{ steps.tag.outputs.tag }}
|
|
name: "Archive (Top ${{ env.DOMAIN_COUNT }}) - ${{ steps.tag.outputs.tag }}"
|
|
body: |
|
|
This release contains favicons for the top **${{ env.DOMAIN_COUNT }}** domains.
|
|
|
|
## Contents
|
|
- **archive.tzst**: Compressed archive containing favicon images for top ${{ env.DOMAIN_COUNT }} domains
|
|
|
|
## Usage
|
|
Extract the archive using the [tzst](https://github.com/xixu-me/tzst):
|
|
```bash
|
|
pip install tzst
|
|
tzst x archive.tzst
|
|
```
|
|
|
|
## Note
|
|
This workflow is configurable! You can manually trigger it with different domain counts (200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, or 200000) using the "Run workflow" button.
|
|
files: |
|
|
archive.tzst
|
|
draft: false
|
|
prerelease: false
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|