diff --git a/scripts/find_redundant_domain.py b/scripts/find_redundant_domain.py new file mode 100644 index 0000000..1f3836d --- /dev/null +++ b/scripts/find_redundant_domain.py @@ -0,0 +1,41 @@ +import sys + + +def load(list): + results = [] + with open(list, "r") as f: + for line in f.readlines(): + line = line.strip() + if line == "" or line.startswith("#"): + continue + domain_labels = line.lower().split(".") + results.append(domain_labels) + results.sort(key=len) + return results + + +def find(labelses, removed_domain_file): + tree = {} + LEAF = 1 + for labels in labelses: + domain = ".".join(labels) + node = tree + while len(labels) > 0: + label = labels.pop() + if label in node: + if node[label] == LEAF: + print(f"Redundant found: {domain} at {'.'.join(labels)}") + with open(removed_domain_file, "a") as f: + f.write(domain) + f.write("\n") + break + else: + if len(labels) == 0: + node[label] = LEAF + else: + node[label] = {} + node = node[label] + + +if __name__ == "__main__": + find(load(sys.argv[1]), sys.argv[2]) diff --git a/scripts/list_to_yaml.py b/scripts/list_to_yaml.py new file mode 100644 index 0000000..f4bcaed --- /dev/null +++ b/scripts/list_to_yaml.py @@ -0,0 +1,18 @@ +import sys + + +def list_to_yaml(input_file, output_file): + with open(input_file, "r", encoding="utf-8") as file: + domains = [f"+.{line.strip()}" for line in file if line.strip()] + yaml_content = "payload:\n" + "\n".join(f" - {domain}" for domain in domains) + with open(output_file, "w", encoding="utf-8") as file: + file.write(yaml_content) + print(f"Processed YAML file saved to {output_file}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(1) + input_file = sys.argv[1] + output_file = sys.argv[2] + list_to_yaml(input_file, output_file) diff --git a/scripts/remove_from.py b/scripts/remove_from.py new file mode 100644 index 0000000..0f15458 --- /dev/null +++ b/scripts/remove_from.py @@ -0,0 +1,21 @@ +import argparse + + +def remove_domains(file_to_remove, file_to_remove_from, output_file): + with open(file_to_remove, "r") as f_remove, open( + file_to_remove_from, "r" + ) as f_from: + domains_to_remove = set(line.strip() for line in f_remove) + all_domains = set(line.strip() for line in f_from) + remaining_domains = all_domains - domains_to_remove + with open(output_file, "w") as output: + output.write("\n".join(remaining_domains)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-remove", required=True, dest="remove_path") + parser.add_argument("-from", required=True, dest="from_path") + parser.add_argument("-out", required=True, dest="out_path") + args = parser.parse_args() + remove_domains(args.remove_path, args.from_path, args.out_path) diff --git a/scripts/yaml_to_list.py b/scripts/yaml_to_list.py new file mode 100644 index 0000000..cf6e270 --- /dev/null +++ b/scripts/yaml_to_list.py @@ -0,0 +1,21 @@ +import sys + +import yaml + + +def yaml_to_list(input_file, output_file): + with open(input_file, "r", encoding="utf-8") as file: + data = yaml.safe_load(file) + domains = [item.lstrip("+.") for item in data.get("payload", [])] + with open(output_file, "w", encoding="utf-8") as file: + for domain in domains: + file.write(f"{domain}\n") + print(f"Processed List file saved to {output_file}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + sys.exit(1) + input_file = sys.argv[1] + output_file = sys.argv[2] + yaml_to_list(input_file, output_file)