Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d293377547 | ||
|
|
1e8a927f9b | ||
|
|
3f07d81f85 | ||
|
|
c1a2e04ba3 | ||
|
|
fa11b36353 | ||
|
|
b35c13d09c | ||
|
|
54e7fdb695 | ||
|
|
e1e3c339c7 | ||
|
|
cc9a12ebc8 | ||
|
|
2939d28e64 | ||
|
|
2e9639457d | ||
|
|
b8f6b82874 | ||
|
|
063dfaa1aa | ||
|
|
0cbfb9f538 | ||
|
|
f1cfff13b0 | ||
|
|
7dcdce9a1c | ||
|
|
4aa7f66fea | ||
|
|
29d9af9975 | ||
|
|
f17483367e | ||
|
|
f584eab3d3 | ||
|
|
fbc2e08152 | ||
|
|
df16f1f920 | ||
|
|
a052b220c2 | ||
|
|
3b60ac5b30 | ||
|
|
10bd59ca3d |
No files matched your search
@@ -0,0 +1,41 @@
|
||||
version: 2
|
||||
updates:
|
||||
# Monitor GitHub Actions dependencies
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "02:00"
|
||||
timezone: "UTC"
|
||||
open-pull-requests-limit: 5
|
||||
reviewers:
|
||||
- "xixu-me"
|
||||
assignees:
|
||||
- "xixu-me"
|
||||
commit-message:
|
||||
prefix: "ci"
|
||||
include: "scope"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "github-actions"
|
||||
|
||||
# Monitor pip dependencies (for tzst package used in workflow)
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
day: "monday"
|
||||
time: "02:30"
|
||||
timezone: "UTC"
|
||||
open-pull-requests-limit: 3
|
||||
reviewers:
|
||||
- "xixu-me"
|
||||
assignees:
|
||||
- "xixu-me"
|
||||
commit-message:
|
||||
prefix: "deps"
|
||||
include: "scope"
|
||||
labels:
|
||||
- "dependencies"
|
||||
- "python"
|
||||
+37
-47
@@ -13,18 +13,20 @@ on:
|
||||
default: "200000"
|
||||
type: string
|
||||
|
||||
# Add permissions for the workflow
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
|
||||
env:
|
||||
# Default domain count (can be overridden by workflow input)
|
||||
DOMAIN_COUNT: ${{ github.event.inputs.domain_count || '200000' }}
|
||||
|
||||
# Global permissions for the workflow
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
|
||||
jobs:
|
||||
validate-input:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Validate domain count input
|
||||
run: |
|
||||
@@ -38,10 +40,13 @@ jobs:
|
||||
fetch-domain-list:
|
||||
runs-on: ubuntu-latest
|
||||
needs: validate-input
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Fetch top domains from Cloudflare Radar API
|
||||
run: |
|
||||
@@ -67,13 +72,16 @@ jobs:
|
||||
process-favicons-and-release:
|
||||
runs-on: ubuntu-latest
|
||||
needs: fetch-domain-list
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Download domain list artifact
|
||||
uses: actions/download-artifact@v4
|
||||
uses: actions/download-artifact@v5
|
||||
with:
|
||||
name: domain-list-artifact
|
||||
path: .
|
||||
@@ -95,9 +103,6 @@ jobs:
|
||||
success_count=0
|
||||
error_count=0
|
||||
|
||||
# Array to track used filenames for duplicate handling
|
||||
declare -A filename_count
|
||||
|
||||
# Read domains file line by line
|
||||
while IFS= read -r domain; do
|
||||
counter=$((counter + 1))
|
||||
@@ -112,62 +117,47 @@ jobs:
|
||||
echo "Progress: $counter/$total_domains domains processed (Success: $success_count, Errors: $error_count)"
|
||||
fi
|
||||
|
||||
# Extract second-level domain (remove top-level domain)
|
||||
# For example: google.com -> google, github.io -> github
|
||||
second_level_domain=$(echo "$domain" | sed 's/\.[^.]*$//')
|
||||
|
||||
# Handle duplicate filenames by adding sequential numbers
|
||||
base_filename="$second_level_domain"
|
||||
if [[ -n "${filename_count[$base_filename]}" ]]; then
|
||||
# Increment count for this filename
|
||||
filename_count[$base_filename]=$((filename_count[$base_filename] + 1))
|
||||
final_filename="${base_filename}_${filename_count[$base_filename]}"
|
||||
else
|
||||
# First occurrence of this filename
|
||||
filename_count[$base_filename]=0
|
||||
final_filename="$base_filename"
|
||||
fi
|
||||
|
||||
# Construct favicon URL
|
||||
favicon_url="https://t1.gstatic.com/faviconV2?client=SOCIAL&type=FAVICON&fallback_opts=TYPE,SIZE,URL&url=https://${domain}&size=256"
|
||||
favicon_file="favicons/${final_filename}.png"
|
||||
|
||||
# Temporarily use a temp file for download
|
||||
temp_favicon_file="favicons/temp_favicon.png"
|
||||
|
||||
# Download favicon with enhanced error handling and timeout
|
||||
if timeout 15 curl -L -s -o "$favicon_file" "$favicon_url" --max-time 10 --retry 3 --retry-delay 1; then
|
||||
if timeout 15 curl -L -s -o "$temp_favicon_file" "$favicon_url" --max-time 10 --retry 3 --retry-delay 1; then
|
||||
# Check if file was actually downloaded and has content
|
||||
if [ -s "$favicon_file" ]; then
|
||||
if [ -s "$temp_favicon_file" ]; then
|
||||
# Calculate SHA-256 hash of the downloaded file
|
||||
file_hash=$(sha256sum "$favicon_file" | cut -d' ' -f1)
|
||||
file_hash=$(sha256sum "$temp_favicon_file" | cut -d' ' -f1)
|
||||
|
||||
# Check if the hash matches the failure hash
|
||||
if [ "$file_hash" = "59bfe9bc385ad69f50793ce4a53397316d7a875a7148a63c16df9b674c6cda64" ]; then
|
||||
# This is a failed download (generic/default favicon)
|
||||
rm -f "$favicon_file"
|
||||
rm -f "$temp_favicon_file"
|
||||
error_count=$((error_count + 1))
|
||||
echo "Warning: Generic favicon detected for domain: $domain (mapped to $final_filename) (SHA-256: $file_hash)"
|
||||
echo "Warning: Generic favicon detected for domain: $domain (SHA-256: $file_hash)"
|
||||
else
|
||||
# Valid favicon downloaded - use domain as filename
|
||||
favicon_file="favicons/${domain}.png"
|
||||
mv "$temp_favicon_file" "$favicon_file"
|
||||
|
||||
# Valid favicon downloaded
|
||||
success_count=$((success_count + 1))
|
||||
echo "Success: Downloaded favicon for $domain -> $final_filename.png"
|
||||
echo "Success: Downloaded favicon for $domain -> ${domain}.png"
|
||||
fi
|
||||
else
|
||||
# Remove empty file
|
||||
rm -f "$favicon_file"
|
||||
rm -f "$temp_favicon_file"
|
||||
error_count=$((error_count + 1))
|
||||
echo "Warning: Empty file for domain: $domain (mapped to $final_filename)"
|
||||
echo "Warning: Empty file for domain: $domain"
|
||||
fi
|
||||
else
|
||||
# Remove any partial file
|
||||
rm -f "$favicon_file"
|
||||
rm -f "$temp_favicon_file"
|
||||
error_count=$((error_count + 1))
|
||||
echo "Error: Failed to download favicon for domain: $domain (mapped to $final_filename)"
|
||||
fi
|
||||
|
||||
# Add high failure rate detection (stop if >80% failures after 100 domains)
|
||||
if [ $counter -gt 100 ] && [ $((error_count * 100 / counter)) -gt 80 ]; then
|
||||
echo "Error: High failure rate detected (${error_count}/${counter}). Stopping execution."
|
||||
exit 1
|
||||
echo "Error: Failed to download favicon for domain: $domain"
|
||||
fi
|
||||
|
||||
done < domains
|
||||
|
||||
echo "Favicon download completed!"
|
||||
@@ -201,9 +191,9 @@ jobs:
|
||||
git push origin assets --force
|
||||
fi
|
||||
|
||||
- name: Install tzst compression tool
|
||||
- name: Install tzst
|
||||
run: |
|
||||
echo "Installing tzst compression tool..."
|
||||
echo "Installing tzst..."
|
||||
pip install tzst
|
||||
tzst --version
|
||||
|
||||
@@ -231,7 +221,7 @@ jobs:
|
||||
echo "Generated release tag: $TAG"
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ steps.tag.outputs.tag }}
|
||||
name: "Archive (Top ${{ env.DOMAIN_COUNT }}) - ${{ steps.tag.outputs.tag }}"
|
||||
|
||||
@@ -14,15 +14,21 @@ This repository hosts a collection of favicon images for the world's most popula
|
||||
|
||||
### Access Individual Favicons
|
||||
|
||||
Individual favicons are stored in the `assets` branch and are named according to their second-level domain:
|
||||
Individual favicons are stored in the `assets` branch and are named according to their domain name:
|
||||
|
||||
- `google.png` for google.com
|
||||
- `github.png` for github.com
|
||||
- `google.com.png` for google.com
|
||||
- `github.com.png` for github.com
|
||||
|
||||
Access patterns:
|
||||
|
||||
```
|
||||
https://raw.githubusercontent.com/xixu-me/favicons/assets/DOMAIN_NAME.png
|
||||
https://raw.githubusercontent.com/xixu-me/favicons/refs/heads/assets/favicons/DOMAIN_NAME.png
|
||||
```
|
||||
|
||||
For better performance and global availability, you can access the favicons through jsDelivr CDN:
|
||||
|
||||
```
|
||||
https://cdn.jsdelivr.net/gh/xixu-me/favicons@refs/heads/assets/favicons/DOMAIN_NAME.png
|
||||
```
|
||||
|
||||
### Download Compressed Archives
|
||||
@@ -46,7 +52,7 @@ tzst x archive.tzst
|
||||
|
||||
Archives are available with the following domain counts:
|
||||
|
||||
- 200 (minimal)
|
||||
- 200
|
||||
- 500
|
||||
- 1,000
|
||||
- 2,000
|
||||
@@ -56,8 +62,6 @@ Archives are available with the following domain counts:
|
||||
- 50,000
|
||||
- 100,000
|
||||
- 200,000
|
||||
- 500,000
|
||||
- 1,000,000 (complete set)
|
||||
|
||||
## Self-Build Instructions
|
||||
|
||||
@@ -110,15 +114,12 @@ You can modify the run.yml file to change:
|
||||
|
||||
### File Naming
|
||||
|
||||
Favicons are named based on the second-level domain:
|
||||
Favicons are named using the domain name:
|
||||
|
||||
- `example.com` → `example.png`
|
||||
- `subdomain.example.com` → `example.png`
|
||||
- `example.com` → `example.com.png`
|
||||
- `subdomain.example.com` → `subdomain.example.com.png`
|
||||
|
||||
When duplicate second-level domains exist, sequential numbers are added:
|
||||
|
||||
- First `github.io` domain → `github.png`
|
||||
- Second `github.io` domain → `github_1.png`
|
||||
This ensures unique filenames for all domains and preserves the complete domain information.
|
||||
|
||||
### Error Handling
|
||||
|
||||
|
||||
Reference in new issue
Block a user