commit 332d08641c3cc37a455c9acee67e0bd725ad5852 Author: Xi Xu Date: Mon Jun 9 21:56:22 2025 +0800 Initial commit diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml new file mode 100644 index 000000000..c9106f43b --- /dev/null +++ b/.github/workflows/run.yml @@ -0,0 +1,258 @@ +name: The Workflow + +on: + # Weekly trigger every Sunday at 02:00 UTC + schedule: + - cron: "0 2 * * 0" + # Manual trigger with configurable domain count + workflow_dispatch: + inputs: + domain_count: + description: "Number of top domains to process (supported values: 200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, 1000000)" + required: false + default: "1000000" + type: string + +# Add permissions for the workflow +permissions: + contents: write + actions: read + +env: + # Default domain count (can be overridden by workflow input) + DOMAIN_COUNT: ${{ github.event.inputs.domain_count || '1000000' }} + +jobs: + validate-input: + runs-on: ubuntu-latest + steps: + - name: Validate domain count input + run: | + VALID_COUNTS="200 500 1000 2000 5000 10000 20000 50000 100000 200000 500000 1000000" + if [[ ! " $VALID_COUNTS " =~ " ${{ env.DOMAIN_COUNT }} " ]]; then + echo "Error: Invalid domain count '${{ env.DOMAIN_COUNT }}'. Supported values: $VALID_COUNTS" + exit 1 + fi + echo "Domain count validation passed: ${{ env.DOMAIN_COUNT }}" + + fetch-domain-list: + runs-on: ubuntu-latest + needs: validate-input + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Fetch top domains from Cloudflare Radar API + run: | + echo "Fetching top ${{ env.DOMAIN_COUNT }} domains from Cloudflare Radar API..." + curl "https://api.cloudflare.com/client/v4/radar/datasets/ranking_top_${{ env.DOMAIN_COUNT }}" \ + --header "Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}" \ + --output domains_raw.txt + + - name: Process domain list + run: | + echo "Processing domain list - removing first line..." + # Remove the first line and save to domains file + tail -n +2 domains_raw.txt > domains + echo "Domain list processed. Total domains: $(wc -l < domains)" + + - name: Upload domain list as artifact + uses: actions/upload-artifact@v4 + with: + name: domain-list-artifact + path: domains + retention-days: 7 + + process-favicons-and-release: + runs-on: ubuntu-latest + needs: fetch-domain-list + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download domain list artifact + uses: actions/download-artifact@v4 + with: + name: domain-list-artifact + path: . + + - name: Create favicons directory + run: | + mkdir -p favicons + echo "Created favicons directory" + + - name: Download favicons for each domain + run: | + echo "Starting favicon download process..." + echo "Processing top ${{ env.DOMAIN_COUNT }} domains..." + total_domains=$(wc -l < domains) + echo "Total domains to process: $total_domains" + + # Counter for progress tracking + counter=0 + success_count=0 + error_count=0 + + # Array to track used filenames for duplicate handling + declare -A filename_count + + # Read domains file line by line + while IFS= read -r domain; do + counter=$((counter + 1)) + + # Skip empty lines + if [ -z "$domain" ]; then + continue + fi + + # Progress indicator every 1000 domains + if [ $((counter % 1000)) -eq 0 ]; then + echo "Progress: $counter/$total_domains domains processed (Success: $success_count, Errors: $error_count)" + fi + + # Extract second-level domain (remove top-level domain) + # For example: google.com -> google, github.io -> github + second_level_domain=$(echo "$domain" | sed 's/\.[^.]*$//') + + # Handle duplicate filenames by adding sequential numbers + base_filename="$second_level_domain" + if [[ -n "${filename_count[$base_filename]}" ]]; then + # Increment count for this filename + filename_count[$base_filename]=$((filename_count[$base_filename] + 1)) + final_filename="${base_filename}_${filename_count[$base_filename]}" + else + # First occurrence of this filename + filename_count[$base_filename]=0 + final_filename="$base_filename" + fi + + # Construct favicon URL + favicon_url="https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://${domain}&size=256" + favicon_file="favicons/${final_filename}.png" + + # Download favicon with enhanced error handling and timeout + if timeout 15 curl -L -s -o "$favicon_file" "$favicon_url" --max-time 10 --retry 3 --retry-delay 1; then + # Check if file was actually downloaded and has content + if [ -s "$favicon_file" ]; then + # Calculate SHA-256 hash of the downloaded file + file_hash=$(sha256sum "$favicon_file" | cut -d' ' -f1) + + # Check if the hash matches the failure hash + if [ "$file_hash" = "59bfe9bc385ad69f50793ce4a53397316d7a875a7148a63c16df9b674c6cda64" ]; then + # This is a failed download (generic/default favicon) + rm -f "$favicon_file" + error_count=$((error_count + 1)) + echo "Warning: Generic favicon detected for domain: $domain (mapped to $final_filename) (SHA-256: $file_hash)" + else + # Valid favicon downloaded + success_count=$((success_count + 1)) + echo "Success: Downloaded favicon for $domain -> $final_filename.png" + fi + else + # Remove empty file + rm -f "$favicon_file" + error_count=$((error_count + 1)) + echo "Warning: Empty file for domain: $domain (mapped to $final_filename)" + fi + else + # Remove any partial file + rm -f "$favicon_file" + error_count=$((error_count + 1)) + echo "Error: Failed to download favicon for domain: $domain (mapped to $final_filename)" + fi + + # Add high failure rate detection (stop if >80% failures after 100 domains) + if [ $counter -gt 100 ] && [ $((error_count * 100 / counter)) -gt 80 ]; then + echo "Error: High failure rate detected (${error_count}/${counter}). Stopping execution." + exit 1 + fi + done < domains + + echo "Favicon download completed!" + echo "Total processed: $counter" + echo "Successful downloads: $success_count" + echo "Failed downloads: $error_count" + echo "Success rate: $((success_count * 100 / counter))%" + echo "Downloaded favicons saved in: $(ls favicons | wc -l) files" + + - name: Push favicons to branch + run: | + # Configure git + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + # Create or switch to assets branch + git checkout --orphan assets + git rm -rf . 2>/dev/null || true + + # Copy favicons directory contents + cp -r favicons/* . 2>/dev/null || echo "No favicons to copy" + + # Add all favicon files + git add *.png 2>/dev/null || echo "No PNG files to add" + + # Commit with timestamp + if git diff --staged --quiet; then + echo "No favicon changes to commit" + else + git commit -m "Update favicons (top ${{ env.DOMAIN_COUNT }}) - $(date -u '+%Y-%m-%d %H:%M:%S UTC')" + git push origin assets --force + fi + + - name: Install tzst compression tool + run: | + echo "Installing tzst compression tool..." + pip install tzst + tzst --version + + - name: Create compressed archive + run: | + echo "Creating compressed favicon archive..." + # Create archive with tzst + tzst a archive.tzst favicons + + # Verify archive was created + if [ -f "archive.tzst" ]; then + echo "Archive created successfully" + ls -lh archive.tzst + else + echo "Error: Archive creation failed" + exit 1 + fi + + - name: Generate release tag + id: tag + run: | + # Generate tag based on current date and domain count + TAG="$(date -u '+%Y-%m-%d')-${{ env.DOMAIN_COUNT }}" + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "Generated release tag: $TAG" + + - name: Create GitHub Release + uses: softprops/action-gh-release@v1 + with: + tag_name: ${{ steps.tag.outputs.tag }} + name: "Archive (Top ${{ env.DOMAIN_COUNT }}) - ${{ steps.tag.outputs.tag }}" + body: | + This release contains favicons for the top **${{ env.DOMAIN_COUNT }}** domains. + + ## Contents + - **archive.tzst**: Compressed archive containing favicon images for top ${{ env.DOMAIN_COUNT }} domains + + ## Usage + Extract the archive using the [tzst](https://github.com/xixu-me/tzst): + ```bash + pip install tzst + tzst x archive.tzst + ``` + + ## Note + This workflow is configurable! You can manually trigger it with different domain counts (200, 500, 1000, 2000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, or 1000000) using the "Run workflow" button. + files: | + archive.tzst + draft: false + prerelease: false + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..99f5cea12 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Xi Xu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 000000000..8aea437f6 --- /dev/null +++ b/README.md @@ -0,0 +1,167 @@ +# Favicons + +A repository of favicons for the top domains on the internet, automatically updated on a weekly basis. + +## Overview + +This repository hosts a collection of favicon images for the world's most popular domains, sourced from Google's favicon service. The collection is refreshed weekly and available in several formats: + +- Individual PNG files in the `assets` branch +- Compressed archives of various sizes in the Releases section +- Custom builds via manual workflow triggers + +## Direct Usage + +### Access Individual Favicons + +Individual favicons are stored in the `assets` branch and are named according to their second-level domain: + +- `google.png` for google.com +- `github.png` for github.com + +Access patterns: + +``` +https://raw.githubusercontent.com/xixu-me/favicons/assets/DOMAIN_NAME.png +``` + +### Download Compressed Archives + +Pre-built archives are available in the [Releases](https://github.com/xixu-me/favicons/releases) section: + +1. Navigate to the Releases page +2. Find the desired release (named by date and domain count) +3. Download the `archive.tzst` file +4. Extract using the [tzst](https://github.com/xixu-me/tzst): + +```bash +# Install tzst if you don't have it +pip install tzst + +# Extract the archive +tzst x archive.tzst +``` + +### Available Archive Sizes + +Archives are available with the following domain counts: + +- 200 (minimal) +- 500 +- 1,000 +- 2,000 +- 5,000 +- 10,000 +- 20,000 +- 50,000 +- 100,000 +- 200,000 +- 500,000 +- 1,000,000 (complete set) + +## Self-Build Instructions + +### Fork the Repository + +1. Click the Fork button at the top right of this repository +2. Clone your forked repository: + + ```bash + git clone https://github.com/xixu-me/favicons.git + cd favicons + ``` + +### Configure Required Secrets + +This workflow requires a Cloudflare API token to access the domain ranking data: + +1. Go to your fork's Settings > Secrets and variables > Actions +2. Create a new repository secret: + - Name: `CLOUDFLARE_API_TOKEN` + - Value: Your Cloudflare API token with Radar access + +### Run the Workflow Manually + +1. Go to the Actions tab in your repository +2. Select "The Workflow" from the list of workflows +3. Click "Run workflow" +4. Select the desired domain count from the dropdown +5. Click "Run workflow" to start the build + +The workflow will: + +1. Validate your input +2. Fetch the top domains from Cloudflare Radar +3. Download favicons for each domain +4. Commit the favicons to the `assets` branch +5. Create a compressed archive +6. Create a release with the archive + +### Customize the Workflow + +You can modify the run.yml file to change: + +- The schedule (currently weekly on Sundays) +- The favicon source URL +- The error handling logic +- The naming conventions + +## Technical Details + +### File Naming + +Favicons are named based on the second-level domain: + +- `example.com` → `example.png` +- `subdomain.example.com` → `example.png` + +When duplicate second-level domains exist, sequential numbers are added: + +- First `github.io` domain → `github.png` +- Second `github.io` domain → `github_1.png` + +### Error Handling + +The workflow includes: + +- Timeout handling (15 seconds per favicon) +- Retry logic (3 attempts) +- Generic/default favicon detection +- Empty file detection +- High failure rate detection (stops if >80% of downloads fail) + +## Disclaimers + +### Copyright Notice + +The favicon images are the property of their respective domain owners. This repository simply provides an aggregation service for easier access and does not claim ownership of any favicon images. + +### Rate Limiting + +This repository fetches favicons from Google's favicon service, which may impose rate limits. If you're running your own build and encounter high failure rates, try reducing the domain count or adding delays between requests. + +### Data Accuracy + +The top domains list is provided by Cloudflare Radar and represents their assessment of domain popularity. Domain rankings change over time and may not reflect the most current web traffic patterns. + +### API Requirements + +Running your own builds requires: + +- A Cloudflare account +- API access to Cloudflare Radar +- A valid API token with appropriate permissions + +## License + +This project's code is licensed under the [MIT License](LICENSE). This license applies only to the code in this repository and not to the favicon images themselves. + +## Acknowledgements + +- Cloudflare Radar for providing domain ranking data +- Google's favicon service for providing the favicon images +- [tzst](https://github.com/xixu-me/tzst) for efficient compression + +--- + +**Note**: This project is not affiliated with or endorsed by Cloudflare, Google, or any domain owners whose favicons are collected.