249 lines
8.8 KiB
YAML
249 lines
8.8 KiB
YAML
name: The Workflow
|
|
|
|
on:
|
|
# Weekly trigger every Sunday at 02:00 UTC
|
|
schedule:
|
|
- cron: "0 2 * * 0"
|
|
# Manual trigger with configurable domain count
|
|
workflow_dispatch:
|
|
inputs:
|
|
domain_count:
|
|
description: "Number of top domains to process (supported values: 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000)"
|
|
required: false
|
|
default: "200000"
|
|
type: string
|
|
|
|
env:
|
|
# Default domain count (can be overridden by workflow input)
|
|
DOMAIN_COUNT: ${{ github.event.inputs.domain_count || '200000' }}
|
|
|
|
# Global permissions for the workflow
|
|
permissions:
|
|
contents: write
|
|
actions: write
|
|
|
|
jobs:
|
|
validate-input:
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
steps:
|
|
- name: Validate domain count input
|
|
run: |
|
|
VALID_COUNTS="200 500 1000 2000 5000 10000 20000 50000 100000 200000"
|
|
if [[ ! " $VALID_COUNTS " =~ " ${{ env.DOMAIN_COUNT }} " ]]; then
|
|
echo "Error: Invalid domain count '${{ env.DOMAIN_COUNT }}'. Supported values: $VALID_COUNTS"
|
|
exit 1
|
|
fi
|
|
echo "Domain count validation passed: ${{ env.DOMAIN_COUNT }}"
|
|
|
|
fetch-domain-list:
|
|
runs-on: ubuntu-latest
|
|
needs: validate-input
|
|
permissions:
|
|
contents: read
|
|
actions: write
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Fetch top domains from Cloudflare Radar API
|
|
run: |
|
|
echo "Fetching top ${{ env.DOMAIN_COUNT }} domains from Cloudflare Radar API..."
|
|
curl "https://api.cloudflare.com/client/v4/radar/datasets/ranking_top_${{ env.DOMAIN_COUNT }}" \
|
|
--header "Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}" \
|
|
--output domains_raw.txt
|
|
|
|
- name: Process domain list
|
|
run: |
|
|
echo "Processing domain list - removing first line..."
|
|
# Remove the first line and save to domains file
|
|
tail -n +2 domains_raw.txt > domains
|
|
echo "Domain list processed. Total domains: $(wc -l < domains)"
|
|
|
|
- name: Upload domain list as artifact
|
|
uses: actions/upload-artifact@v5
|
|
with:
|
|
name: domain-list-artifact
|
|
path: domains
|
|
retention-days: 7
|
|
|
|
process-favicons-and-release:
|
|
runs-on: ubuntu-latest
|
|
needs: fetch-domain-list
|
|
permissions:
|
|
contents: write
|
|
actions: read
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Download domain list artifact
|
|
uses: actions/download-artifact@v6
|
|
with:
|
|
name: domain-list-artifact
|
|
path: .
|
|
|
|
- name: Create favicons directory
|
|
run: |
|
|
mkdir -p favicons
|
|
echo "Created favicons directory"
|
|
|
|
- name: Download favicons for each domain
|
|
run: |
|
|
echo "Starting favicon download process..."
|
|
echo "Processing top ${{ env.DOMAIN_COUNT }} domains..."
|
|
total_domains=$(wc -l < domains)
|
|
echo "Total domains to process: $total_domains"
|
|
|
|
# Counter for progress tracking
|
|
counter=0
|
|
success_count=0
|
|
error_count=0
|
|
|
|
# Read domains file line by line
|
|
while IFS= read -r domain; do
|
|
counter=$((counter + 1))
|
|
|
|
# Skip empty lines
|
|
if [ -z "$domain" ]; then
|
|
continue
|
|
fi
|
|
|
|
# Progress indicator every 1000 domains
|
|
if [ $((counter % 1000)) -eq 0 ]; then
|
|
echo "Progress: $counter/$total_domains domains processed (Success: $success_count, Errors: $error_count)"
|
|
fi
|
|
|
|
# Construct favicon URL
|
|
favicon_url="https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://${domain}&size=256"
|
|
|
|
# Temporarily use a temp file for download
|
|
temp_favicon_file="favicons/temp_favicon.png"
|
|
|
|
# Download favicon with enhanced error handling and timeout
|
|
if timeout 15 curl -L -s -o "$temp_favicon_file" "$favicon_url" --max-time 10 --retry 3 --retry-delay 1; then
|
|
# Check if file was actually downloaded and has content
|
|
if [ -s "$temp_favicon_file" ]; then
|
|
# Calculate SHA-256 hash of the downloaded file
|
|
file_hash=$(sha256sum "$temp_favicon_file" | cut -d' ' -f1)
|
|
|
|
# Check if the hash matches the failure hash
|
|
if [ "$file_hash" = "59bfe9bc385ad69f50793ce4a53397316d7a875a7148a63c16df9b674c6cda64" ]; then
|
|
# This is a failed download (generic/default favicon)
|
|
rm -f "$temp_favicon_file"
|
|
error_count=$((error_count + 1))
|
|
echo "Warning: Generic favicon detected for domain: $domain (SHA-256: $file_hash)"
|
|
else
|
|
# Valid favicon downloaded - use domain as filename
|
|
favicon_file="favicons/${domain}.png"
|
|
mv "$temp_favicon_file" "$favicon_file"
|
|
|
|
# Valid favicon downloaded
|
|
success_count=$((success_count + 1))
|
|
echo "Success: Downloaded favicon for $domain -> ${domain}.png"
|
|
fi
|
|
else
|
|
# Remove empty file
|
|
rm -f "$temp_favicon_file"
|
|
error_count=$((error_count + 1))
|
|
echo "Warning: Empty file for domain: $domain"
|
|
fi
|
|
else
|
|
# Remove any partial file
|
|
rm -f "$temp_favicon_file"
|
|
error_count=$((error_count + 1))
|
|
echo "Error: Failed to download favicon for domain: $domain"
|
|
fi
|
|
|
|
done < domains
|
|
|
|
echo "Favicon download completed!"
|
|
echo "Total processed: $counter"
|
|
echo "Successful downloads: $success_count"
|
|
echo "Failed downloads: $error_count"
|
|
echo "Success rate: $((success_count * 100 / counter))%"
|
|
echo "Downloaded favicons saved in: $(ls favicons | wc -l) files"
|
|
|
|
- name: Push favicons to branch
|
|
run: |
|
|
# Configure git
|
|
git config --global user.name 'github-actions[bot]'
|
|
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
|
|
|
# Create or switch to assets branch
|
|
git checkout --orphan assets
|
|
git rm -rf . 2>/dev/null || true
|
|
|
|
# Copy favicons directory contents
|
|
cp -r favicons/* . 2>/dev/null || echo "No favicons to copy"
|
|
|
|
# Add all favicon files
|
|
git add *.png 2>/dev/null || echo "No PNG files to add"
|
|
|
|
# Commit with timestamp
|
|
if git diff --staged --quiet; then
|
|
echo "No favicon changes to commit"
|
|
else
|
|
git commit -m "Update favicons (top ${{ env.DOMAIN_COUNT }}) - $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
|
git push origin assets --force
|
|
fi
|
|
|
|
- name: Install tzst
|
|
run: |
|
|
echo "Installing tzst..."
|
|
pip install tzst
|
|
tzst --version
|
|
|
|
- name: Create compressed archive
|
|
run: |
|
|
echo "Creating compressed favicon archive..."
|
|
# Create archive with tzst
|
|
tzst a archive.tzst favicons
|
|
|
|
# Verify archive was created
|
|
if [ -f "archive.tzst" ]; then
|
|
echo "Archive created successfully"
|
|
ls -lh archive.tzst
|
|
else
|
|
echo "Error: Archive creation failed"
|
|
exit 1
|
|
fi
|
|
|
|
- name: Generate release tag
|
|
id: tag
|
|
run: |
|
|
# Generate tag based on current date and domain count
|
|
TAG="$(date -u '+%Y-%m-%d')-${{ env.DOMAIN_COUNT }}"
|
|
echo "tag=$TAG" >> $GITHUB_OUTPUT
|
|
echo "Generated release tag: $TAG"
|
|
|
|
- name: Create GitHub Release
|
|
uses: softprops/action-gh-release@v2
|
|
with:
|
|
tag_name: ${{ steps.tag.outputs.tag }}
|
|
name: "Archive (Top ${{ env.DOMAIN_COUNT }}) - ${{ steps.tag.outputs.tag }}"
|
|
body: |
|
|
This release contains favicons for the top **${{ env.DOMAIN_COUNT }}** domains.
|
|
|
|
## Contents
|
|
- **archive.tzst**: Compressed archive containing favicon images for top ${{ env.DOMAIN_COUNT }} domains
|
|
|
|
## Usage
|
|
Extract the archive using the [tzst](https://github.com/xixu-me/tzst):
|
|
```bash
|
|
pip install tzst
|
|
tzst x archive.tzst
|
|
```
|
|
|
|
## Note
|
|
This workflow is configurable! You can manually trigger it with different domain counts (200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, or 200000) using the "Run workflow" button.
|
|
files: |
|
|
archive.tzst
|
|
draft: false
|
|
prerelease: false
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|