xixu-me/favicons
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
xixu-me committed 2025-06-09 21:56:22 +08:00
commit 332d08641c
3 files changed
+446
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/run.yml
+258
View File
@@ -0,0 +1,258 @@
name: The Workflow
on:
# Weekly trigger every Sunday at 02:00 UTC
schedule:
- cron: "0 2 * * 0"
# Manual trigger with configurable domain count
workflow_dispatch:
inputs:
domain_count:
description: "Number of top domains to process (supported values: 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000)"
required: false
default: "1000000"
type: string
# Add permissions for the workflow
permissions:
contents: write
actions: read
env:
# Default domain count (can be overridden by workflow input)
DOMAIN_COUNT: ${{ github.event.inputs.domain_count || '1000000' }}
jobs:
validate-input:
runs-on: ubuntu-latest
steps:
- name: Validate domain count input
run: |
VALID_COUNTS="200 500 1000 2000 5000 10000 20000 50000 100000 200000 500000 1000000"
if [[ ! " $VALID_COUNTS " =~ " ${{ env.DOMAIN_COUNT }} " ]]; then
echo "Error: Invalid domain count '${{ env.DOMAIN_COUNT }}'. Supported values: $VALID_COUNTS"
exit 1
fi
echo "Domain count validation passed: ${{ env.DOMAIN_COUNT }}"
fetch-domain-list:
runs-on: ubuntu-latest
needs: validate-input
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Fetch top domains from Cloudflare Radar API
run: |
echo "Fetching top ${{ env.DOMAIN_COUNT }} domains from Cloudflare Radar API..."
curl "https://api.cloudflare.com/client/v4/radar/datasets/ranking_top_${{ env.DOMAIN_COUNT }}" \
--header "Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}" \
--output domains_raw.txt
- name: Process domain list
run: |
echo "Processing domain list - removing first line..."
# Remove the first line and save to domains file
tail -n +2 domains_raw.txt > domains
echo "Domain list processed. Total domains: $(wc -l < domains)"
- name: Upload domain list as artifact
uses: actions/upload-artifact@v4
with:
name: domain-list-artifact
path: domains
retention-days: 7
process-favicons-and-release:
runs-on: ubuntu-latest
needs: fetch-domain-list
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Download domain list artifact
uses: actions/download-artifact@v4
with:
name: domain-list-artifact
path: .
- name: Create favicons directory
run: |
mkdir -p favicons
echo "Created favicons directory"
- name: Download favicons for each domain
run: |
echo "Starting favicon download process..."
echo "Processing top ${{ env.DOMAIN_COUNT }} domains..."
total_domains=$(wc -l < domains)
echo "Total domains to process: $total_domains"
# Counter for progress tracking
counter=0
success_count=0
error_count=0
# Array to track used filenames for duplicate handling
declare -A filename_count
# Read domains file line by line
while IFS= read -r domain; do
counter=$((counter + 1))
# Skip empty lines
if [ -z "$domain" ]; then
continue
fi
# Progress indicator every 1000 domains
if [ $((counter % 1000)) -eq 0 ]; then
echo "Progress: $counter/$total_domains domains processed (Success: $success_count, Errors: $error_count)"
fi
# Extract second-level domain (remove top-level domain)
# For example: google.com -> google, github.io -> github
second_level_domain=$(echo "$domain" | sed 's/\.[^.]*$//')
# Handle duplicate filenames by adding sequential numbers
base_filename="$second_level_domain"
if [[ -n "${filename_count[$base_filename]}" ]]; then
# Increment count for this filename
filename_count[$base_filename]=$((filename_count[$base_filename] + 1))
final_filename="${base_filename}_${filename_count[$base_filename]}"
else
# First occurrence of this filename
filename_count[$base_filename]=0
final_filename="$base_filename"
fi
# Construct favicon URL
favicon_url="https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://${domain}&size=256"
favicon_file="favicons/${final_filename}.png"
# Download favicon with enhanced error handling and timeout
if timeout 15 curl -L -s -o "$favicon_file" "$favicon_url" --max-time 10 --retry 3 --retry-delay 1; then
# Check if file was actually downloaded and has content
if [ -s "$favicon_file" ]; then
# Calculate SHA-256 hash of the downloaded file
file_hash=$(sha256sum "$favicon_file" | cut -d' ' -f1)
# Check if the hash matches the failure hash
if [ "$file_hash" = "59bfe9bc385ad69f50793ce4a53397316d7a875a7148a63c16df9b674c6cda64" ]; then
# This is a failed download (generic/default favicon)
rm -f "$favicon_file"
error_count=$((error_count + 1))
echo "Warning: Generic favicon detected for domain: $domain (mapped to $final_filename) (SHA-256: $file_hash)"
else
# Valid favicon downloaded
success_count=$((success_count + 1))
echo "Success: Downloaded favicon for $domain -> $final_filename.png"
fi
else
# Remove empty file
rm -f "$favicon_file"
error_count=$((error_count + 1))
echo "Warning: Empty file for domain: $domain (mapped to $final_filename)"
fi
else
# Remove any partial file
rm -f "$favicon_file"
error_count=$((error_count + 1))
echo "Error: Failed to download favicon for domain: $domain (mapped to $final_filename)"
fi
# Add high failure rate detection (stop if >80% failures after 100 domains)
if [ $counter -gt 100 ] && [ $((error_count * 100 / counter)) -gt 80 ]; then
echo "Error: High failure rate detected (${error_count}/${counter}). Stopping execution."
exit 1
fi
done < domains
echo "Favicon download completed!"
echo "Total processed: $counter"
echo "Successful downloads: $success_count"
echo "Failed downloads: $error_count"
echo "Success rate: $((success_count * 100 / counter))%"
echo "Downloaded favicons saved in: $(ls favicons | wc -l) files"
- name: Push favicons to branch
run: |
# Configure git
git config --global user.name 'github-actions[bot]'
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
# Create or switch to assets branch
git checkout --orphan assets
git rm -rf . 2>/dev/null || true
# Copy favicons directory contents
cp -r favicons/* . 2>/dev/null || echo "No favicons to copy"
# Add all favicon files
git add *.png 2>/dev/null || echo "No PNG files to add"
# Commit with timestamp
if git diff --staged --quiet; then
echo "No favicon changes to commit"
else
git commit -m "Update favicons (top ${{ env.DOMAIN_COUNT }}) - $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
git push origin assets --force
fi
- name: Install tzst compression tool
run: |
echo "Installing tzst compression tool..."
pip install tzst
tzst --version
- name: Create compressed archive
run: |
echo "Creating compressed favicon archive..."
# Create archive with tzst
tzst a archive.tzst favicons
# Verify archive was created
if [ -f "archive.tzst" ]; then
echo "Archive created successfully"
ls -lh archive.tzst
else
echo "Error: Archive creation failed"
exit 1
fi
- name: Generate release tag
id: tag
run: |
# Generate tag based on current date and domain count
TAG="$(date -u '+%Y-%m-%d')-${{ env.DOMAIN_COUNT }}"
echo "tag=$TAG" >> $GITHUB_OUTPUT
echo "Generated release tag: $TAG"
- name: Create GitHub Release
uses: softprops/action-gh-release@v1
with:
tag_name: ${{ steps.tag.outputs.tag }}
name: "Archive (Top ${{ env.DOMAIN_COUNT }}) - ${{ steps.tag.outputs.tag }}"
body: |
This release contains favicons for the top **${{ env.DOMAIN_COUNT }}** domains.
## Contents
- **archive.tzst**: Compressed archive containing favicon images for top ${{ env.DOMAIN_COUNT }} domains
## Usage
Extract the archive using the [tzst](https://github.com/xixu-me/tzst):
```bash
pip install tzst
tzst x archive.tzst
```
## Note
This workflow is configurable! You can manually trigger it with different domain counts (200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, or 1000000) using the "Run workflow" button.
files: |
archive.tzst
draft: false
prerelease: false
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 Xi Xu
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
README.md
+167
View File
@@ -0,0 +1,167 @@
# Favicons
A repository of favicons for the top domains on the internet, automatically updated on a weekly basis.
## Overview
This repository hosts a collection of favicon images for the world's most popular domains, sourced from Google's favicon service. The collection is refreshed weekly and available in several formats:
- Individual PNG files in the `assets` branch
- Compressed archives of various sizes in the Releases section
- Custom builds via manual workflow triggers
## Direct Usage
### Access Individual Favicons
Individual favicons are stored in the `assets` branch and are named according to their second-level domain:
- `google.png` for google.com
- `github.png` for github.com
Access patterns:
```
https://raw.githubusercontent.com/xixu-me/favicons/assets/DOMAIN_NAME.png
```
### Download Compressed Archives
Pre-built archives are available in the [Releases](https://github.com/xixu-me/favicons/releases) section:
1. Navigate to the Releases page
2. Find the desired release (named by date and domain count)
3. Download the `archive.tzst` file
4. Extract using the [tzst](https://github.com/xixu-me/tzst):
```bash
# Install tzst if you don't have it
pip install tzst
# Extract the archive
tzst x archive.tzst
```
### Available Archive Sizes
Archives are available with the following domain counts:
- 200 (minimal)
- 500
- 1,000
- 2,000
- 5,000
- 10,000
- 20,000
- 50,000
- 100,000
- 200,000
- 500,000
- 1,000,000 (complete set)
## Self-Build Instructions
### Fork the Repository
1. Click the Fork button at the top right of this repository
2. Clone your forked repository:
```bash
git clone https://github.com/xixu-me/favicons.git
cd favicons
```
### Configure Required Secrets
This workflow requires a Cloudflare API token to access the domain ranking data:
1. Go to your fork's Settings > Secrets and variables > Actions
2. Create a new repository secret:
- Name: `CLOUDFLARE_API_TOKEN`
- Value: Your Cloudflare API token with Radar access
### Run the Workflow Manually
1. Go to the Actions tab in your repository
2. Select "The Workflow" from the list of workflows
3. Click "Run workflow"
4. Select the desired domain count from the dropdown
5. Click "Run workflow" to start the build
The workflow will:
1. Validate your input
2. Fetch the top domains from Cloudflare Radar
3. Download favicons for each domain
4. Commit the favicons to the `assets` branch
5. Create a compressed archive
6. Create a release with the archive
### Customize the Workflow
You can modify the run.yml file to change:
- The schedule (currently weekly on Sundays)
- The favicon source URL
- The error handling logic
- The naming conventions
## Technical Details
### File Naming
Favicons are named based on the second-level domain:
- `example.com` → `example.png`
- `subdomain.example.com` → `example.png`
When duplicate second-level domains exist, sequential numbers are added:
- First `github.io` domain → `github.png`
- Second `github.io` domain → `github_1.png`
### Error Handling
The workflow includes:
- Timeout handling (15 seconds per favicon)
- Retry logic (3 attempts)
- Generic/default favicon detection
- Empty file detection
- High failure rate detection (stops if >80% of downloads fail)
## Disclaimers
### Copyright Notice
The favicon images are the property of their respective domain owners. This repository simply provides an aggregation service for easier access and does not claim ownership of any favicon images.
### Rate Limiting
This repository fetches favicons from Google's favicon service, which may impose rate limits. If you're running your own build and encounter high failure rates, try reducing the domain count or adding delays between requests.
### Data Accuracy
The top domains list is provided by Cloudflare Radar and represents their assessment of domain popularity. Domain rankings change over time and may not reflect the most current web traffic patterns.
### API Requirements
Running your own builds requires:
- A Cloudflare account
- API access to Cloudflare Radar
- A valid API token with appropriate permissions
## License
This project's code is licensed under the [MIT License](LICENSE). This license applies only to the code in this repository and not to the favicon images themselves.
## Acknowledgements
- Cloudflare Radar for providing domain ranking data
- Google's favicon service for providing the favicon images
- [tzst](https://github.com/xixu-me/tzst) for efficient compression
---
**Note**: This project is not affiliated with or endorsed by Cloudflare, Google, or any domain owners whose favicons are collected.