includes the habu.ctfr command

.setup.cfg

@@ -0,0 +1,2 @@
+[sdist_dsc]
+depends: python3-bs4

README.rst

@@ -118,6 +118,27 @@ This command tries to connect to various services and check if you can reach the
     HTTPS: True
 
 
+habu.ctfr: Subdomain mapping
+----------------------------
+This command downloads the certificate transparency logs for a domain and check with DNS queries if each 
+subdomain exists.
+
+Uses multithreading to improve the performance of the DNS queries. 
+
+.. code-block:: bash
+
+    $ sudo habu.ctrf securetia.com
+    [
+        "karma.securetia.com.",
+        "www.securetia.com."
+    ]
+    ...
+
+You can disable the DNS verification with the option '-n'.
+
+**Note**: This command it's based on code from https://github.com/UnaPibaGeek/ctfr
+
+
 habu.dhcp_discover: Discover DHCP servers
 -----------------------------------------
 This command send a DHCP request and shows what devices has replied. Using the '-v' parameter (verbose) you can 

build_deb.sh

@@ -2,5 +2,5 @@
 
 python3 setup.py sdist
 rm -Rf deb_dist
-py2dsc-deb $(ls -1r dist/*.gz | head -n 1)
-
+#py2dsc-deb $(ls -1r dist/*.gz | head -n 1)
+python setup.py --command-packages=stdeb.command bdist_deb

habu/cli/cmd_ctfr.py

@@ -0,0 +1,61 @@
+# based on code from https://github.com/UnaPibaGeek/ctfr
+
+import json
+import logging
+
+import click
+import requests
+import requests_cache
+
+from habu.lib.dns import query_bulk
+
+requests_cache.install_cache('/tmp/habu_requests_cache')
+
+@click.command()
+@click.argument('domain')
+@click.option('-c', 'no_cache', is_flag=True, default=False, help='Disable cache')
+@click.option('-n', 'no_validate', is_flag=True, default=False, help='Disable DNS subdomain validation')
+@click.option('-v', 'verbose', is_flag=True, default=False, help='Verbose output')
+def cmd_ctfr(domain, no_cache, no_validate, verbose):
+
+    if verbose:
+        logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+    if not no_cache:
+        requests_cache.install_cache('/tmp/habu_requests_cache')
+
+    subdomains = set()
+
+    req = requests.get("https://crt.sh/?q=%.{d}&output=json".format(d=domain))
+
+    if req.status_code != 200:
+        print("[X] Information not available!")
+        exit(1)
+
+    json_data = json.loads('[{}]'.format(req.text.replace('}{', '},{')))
+
+    for data in json_data:
+        name = data['name_value'].lower()
+        if '*' not in name:
+            subdomains.add(name)
+
+    subdomains = list(subdomains)
+
+    if no_validate:
+        print(json.dumps(sorted(subdomains), indent=4))
+        return True
+
+    answers = query_bulk(subdomains)
+
+    validated = []
+
+    for answer in answers:
+        if answer:
+            validated.append(str(answer.qname))
+
+    print(json.dumps(sorted(validated), indent=4))
+    return True
+
+
+if __name__ == '__main__':
+    cmd_ctfr()

habu/lib/dns.py

@@ -0,0 +1,29 @@
+from habu.lib.tomorrow3 import threads
+import dns.resolver
+import dns.zone
+from time import sleep
+
+
+@threads(50)
+def __threaded_query(hostname):
+    try:
+        answer = dns.resolver.query(hostname)
+        return answer
+    except Exception:
+        return None
+
+
+def query_bulk(names):
+
+    answers = [__threaded_query(name) for name in names]
+
+    while True:
+        if all([ a.done() for a in answers]):
+            break
+        sleep(1)
+
+    return [answer.result() for answer in answers]
+
+    #for answer in answers:
+    #    print(answer.result())
+

habu/lib/tomorrow3.py

@@ -0,0 +1,48 @@
+import sys
+import traceback
+from functools import wraps
+from threading import Semaphore
+from concurrent.futures import ThreadPoolExecutor
+
+
+def threads(n, queue_max=None):
+
+    def decorator(f):
+
+        pool = ThreadPoolExecutor(n)
+        sem_max = queue_max
+
+        if sem_max is None:
+            sem_max = n
+
+        sem = Semaphore(sem_max)
+
+        def wait():
+            for _ in range(sem_max):
+                sem.acquire()
+
+            for _ in range(sem_max):
+                sem.release()
+
+        f.wait = wait
+
+        def exception_catcher(f, *args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except Exception:
+                print('[tomorrow3] Caught exception.', file=sys.stderr)
+                traceback.print_exc()
+                sys.exit(-1)
+
+        @wraps(f)
+        def wrapped(*args, **kwargs):
+
+            sem.acquire(blocking=True)
+            future = pool.submit(exception_catcher, f, *args, **kwargs)
+            future.add_done_callback(lambda _: sem.release())
+
+            return future
+
+        return wrapped
+
+    return decorator

setup.py

@@ -5,15 +5,16 @@
 
 setup(
     name='habu',
-    version='0.0.45',
+    version='0.0.47',
     description='Python Network Hacking Toolkit',
     long_description=readme,
     author='Fabian Martinez Portantier',
     author_email='[email protected]',
     url='https://github.com/portantier/habu',
     license='Copyright Fabian Martinez Portantier',
     install_requires=[
-        'bs4',
+        #'bs4',
+        'beautifulsoup4',
         'click',
         'regex',
         'requests',
@@ -31,6 +32,7 @@
         habu.arpsniff=habu.cli.cmd_arpsniff:cmd_arpsniff
         habu.b64=habu.cli.cmd_b64:cmd_b64
         habu.contest=habu.cli.cmd_contest:cmd_contest
+        habu.ctfr=habu.cli.cmd_ctfr:cmd_ctfr
         habu.dhcp_discover=habu.cli.cmd_dhcp_discover:cmd_dhcp_discover
         habu.dhcp_starvation=habu.cli.cmd_dhcp_starvation:cmd_dhcp_starvation
         habu.eicar=habu.cli.cmd_eicar:cmd_eicar

stdeb.cfg

@@ -1,4 +1,3 @@
 [DEFAULT]
-Depends3: python3-bs4
 Depends: python3-bs4
-
+Depends3: python3-bs4