Create scripts

scripts/find_redundant_domain.py

@@ -0,0 +1,41 @@
+import sys
+
+
+def load(list):
+    results = []
+    with open(list, "r") as f:
+        for line in f.readlines():
+            line = line.strip()
+            if line == "" or line.startswith("#"):
+                continue
+            domain_labels = line.lower().split(".")
+            results.append(domain_labels)
+    results.sort(key=len)
+    return results
+
+
+def find(labelses, removed_domain_file):
+    tree = {}
+    LEAF = 1
+    for labels in labelses:
+        domain = ".".join(labels)
+        node = tree
+        while len(labels) > 0:
+            label = labels.pop()
+            if label in node:
+                if node[label] == LEAF:
+                    print(f"Redundant found: {domain} at {'.'.join(labels)}")
+                    with open(removed_domain_file, "a") as f:
+                        f.write(domain)
+                        f.write("\n")
+                    break
+            else:
+                if len(labels) == 0:
+                    node[label] = LEAF
+                else:
+                    node[label] = {}
+            node = node[label]
+
+
+if __name__ == "__main__":
+    find(load(sys.argv[1]), sys.argv[2])

scripts/list_to_yaml.py

@@ -0,0 +1,18 @@
+import sys
+
+
+def list_to_yaml(input_file, output_file):
+    with open(input_file, "r", encoding="utf-8") as file:
+        domains = [f"+.{line.strip()}" for line in file if line.strip()]
+    yaml_content = "payload:\n" + "\n".join(f"  - {domain}" for domain in domains)
+    with open(output_file, "w", encoding="utf-8") as file:
+        file.write(yaml_content)
+    print(f"Processed YAML file saved to {output_file}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        sys.exit(1)
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    list_to_yaml(input_file, output_file)

scripts/remove_from.py

@@ -0,0 +1,21 @@
+import argparse
+
+
+def remove_domains(file_to_remove, file_to_remove_from, output_file):
+    with open(file_to_remove, "r") as f_remove, open(
+        file_to_remove_from, "r"
+    ) as f_from:
+        domains_to_remove = set(line.strip() for line in f_remove)
+        all_domains = set(line.strip() for line in f_from)
+    remaining_domains = all_domains - domains_to_remove
+    with open(output_file, "w") as output:
+        output.write("\n".join(remaining_domains))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-remove", required=True, dest="remove_path")
+    parser.add_argument("-from", required=True, dest="from_path")
+    parser.add_argument("-out", required=True, dest="out_path")
+    args = parser.parse_args()
+    remove_domains(args.remove_path, args.from_path, args.out_path)

scripts/yaml_to_list.py

@@ -0,0 +1,21 @@
+import sys
+
+import yaml
+
+
+def yaml_to_list(input_file, output_file):
+    with open(input_file, "r", encoding="utf-8") as file:
+        data = yaml.safe_load(file)
+    domains = [item.lstrip("+.") for item in data.get("payload", [])]
+    with open(output_file, "w", encoding="utf-8") as file:
+        for domain in domains:
+            file.write(f"{domain}\n")
+    print(f"Processed List file saved to {output_file}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        sys.exit(1)
+    input_file = sys.argv[1]
+    output_file = sys.argv[2]
+    yaml_to_list(input_file, output_file)