Merge pull request #95 from OSINT-TECHNOLOGIES/rolling

Stabilized v1.1.6
OSINT-TECHNOLOGIES · Dec 28, 2024 · 3a38e00 · 3a38e00
2 parents e010f48 + ed793b0
commit 3a38e00
Show file tree

Hide file tree

Showing 16 changed files with 680 additions and 136 deletions.
diff --git a/README.md b/README.md
@@ -172,7 +172,7 @@ If you have problems with starting installer.sh, you should try to use `dos2unix
 - [x] Rework Google Dorking module in separate mode
 - [x] Rework Google Dorks list into separate databases with different pre-configured dorks for various purposes
 - [x] Allow user to create their own dorks DB
-- [ ] Add separate API search mode with different free APIs
+- [x] Add separate API search mode with different free APIs
 
 # DPULSE mentions in social medias
 

diff --git a/datagather_modules/data_assembler.py b/datagather_modules/data_assembler.py
@@ -147,27 +147,43 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
             api_scan_db = []
             if used_api_flag != ['Empty']:
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
-                if 1 in used_api_flag:
+                if '1' in used_api_flag:
                     vt_cats, vt_deturls, vt_detsamples, vt_undetsamples = api_virustotal_check(short_domain)
                     api_scan_db.append('VirusTotal')
-                if 2 in used_api_flag:
+                if '2' in used_api_flag:
                     st_alexa, st_apex, st_hostname, st_alivesds, st_txt, a_records_list, mx_records_list, ns_records_list, soa_records_list = api_securitytrails_check(short_domain)
                     api_scan_db.append('SecurityTrails')
-                print(Fore.LIGHTMAGENTA_EX + f"[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
+                if '1' not in used_api_flag:
+                    vt_cats = vt_deturls = vt_detsamples = vt_undetsamples = 'No results because user did not selected VirusTotal API scan'
+                if '2' not in used_api_flag:
+                    st_alexa = st_apex = st_hostname = st_alivesds = st_txt = a_records_list = mx_records_list = ns_records_list = soa_records_list = 'No results because user did not selected SecurityTrails API scan'
+                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
             else:
                 vt_cats = vt_deturls = vt_detsamples = vt_undetsamples = 'No results because user did not selected VirusTotal API scan'
                 st_alexa = st_apex = st_hostname = st_alivesds = st_txt = a_records_list = mx_records_list = ns_records_list = soa_records_list = 'No results because user did not selected SecurityTrails API scan'
                 api_scan_db.append('No')
                 pass
 
+            #data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
+            #               subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records,
+            #               robots_txt_result, sitemap_xml_result, sitemap_links_status,
+            #               web_servers, cms, programming_languages, web_frameworks, analytics, javascript_frameworks, ports,
+            #               hostnames, cpes, tags, vulns, common_socials, total_socials, ps_emails_return,
+            #               accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter,
+            #               website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, dorking_status, dorking_results, vt_cats,
+            #               vt_deturls, vt_detsamples, vt_undetsamples]
+
+            cleaned_dorking = [item.strip() for item in dorking_results if item.strip()]
+            #print(cleaned_dorking)
+
             data_array = [ip, res, mails, subdomains, subdomains_amount, social_medias, subdomain_mails, sd_socials,
                           subdomain_ip, issuer, subject, notBefore, notAfter, commonName, serialNumber, mx_records,
                           robots_txt_result, sitemap_xml_result, sitemap_links_status,
                           web_servers, cms, programming_languages, web_frameworks, analytics, javascript_frameworks, ports,
                           hostnames, cpes, tags, vulns, common_socials, total_socials, ps_emails_return,
                           accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter,
-                          website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, dorking_status, dorking_results, vt_cats,
-                          vt_deturls, vt_detsamples, vt_undetsamples]
+                          website_elements_counter, exposed_passwords_counter, total_links_counter, accessed_links_counter, cleaned_dorking,
+                          vt_cats, vt_deturls, vt_detsamples, vt_undetsamples, st_alexa, st_apex, st_hostname, st_alivesds, st_txt, a_records_list, mx_records_list, ns_records_list, soa_records_list]
 
         elif report_file_type == 'html':
             if pagesearch_flag.lower() == 'y':
@@ -207,15 +223,15 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
             api_scan_db = []
             if used_api_flag != ['Empty']:
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
-                if 1 in used_api_flag:
+                if '1' in used_api_flag:
                     vt_cats, vt_deturls, vt_detsamples, vt_undetsamples = api_virustotal_check(short_domain)
                     api_scan_db.append('VirusTotal')
-                if 2 in used_api_flag:
+                if '2' in used_api_flag:
                     st_alexa, st_apex, st_hostname, st_alivesds, st_txt, a_records_list, mx_records_list, ns_records_list, soa_records_list = api_securitytrails_check(short_domain)
                     api_scan_db.append('SecurityTrails')
-                if 1 not in used_api_flag:
+                if '1' not in used_api_flag:
                     vt_cats = vt_deturls = vt_detsamples = vt_undetsamples = 'No results because user did not selected VirusTotal API scan'
-                if 2 not in used_api_flag:
+                if '2' not in used_api_flag:
                     st_alexa = st_apex = st_hostname = st_alivesds = st_txt = a_records_list = mx_records_list = ns_records_list = soa_records_list = 'No results because user did not selected SecurityTrails API scan'
                 print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
             else:

diff --git a/datagather_modules/networking_processor.py b/datagather_modules/networking_processor.py
@@ -54,7 +54,7 @@ def get_ssl_certificate(short_domain, port=443):
     except (ssl.CertificateError, ssl.SSLError, socket.gaierror, ConnectionRefusedError) as e:
         print(Fore.RED + "Error while gathering info about SSL certificate. See journal for details")
         logging.error(f'SSL CERTIFICATE GATHERING: ERROR. REASON: {e}')
-        issuer = subject = notBefore = notAfter = commonName = serialNumber = ["No information about SSL certificate was gathered"]
+        issuer = subject = notBefore = notAfter = commonName = serialNumber = "No information about SSL certificate was gathered"
         return issuer, subject, notBefore, notAfter, commonName, serialNumber
 
 def query_internetdb(ip, report_file_extension):

diff --git a/dpulse.py b/dpulse.py
@@ -109,7 +109,6 @@ def run():
                                 print(Fore.RED + "Entered domain is not accessible. Scan is impossible" + Style.RESET_ALL)
                                 break
                             case_comment = input(Fore.YELLOW + "Enter case comment >> ")
-                            print(Fore.GREEN + "Be advised that PDF report support is deprecated since v1.1.4. Instead, try to work with HTML report :)" + Style.RESET_ALL)
                             report_filetype = input(Fore.YELLOW + "Enter report file extension [XLSX/HTML] >> ")
                             if not report_filetype:
                                 print(Fore.RED + "\nReport filetype cannot be empty")

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "dpulse"
-version = "1.1.5"
+version = "1.1.6"
 description = "Convenient,fast and user-friendly collector of domain information from Open-Sources"
 authors = ["OSINT-TECHNOLOGIES <[email protected]>"]
 readme = "README.md"
@@ -22,7 +22,7 @@ classifiers = [
 
 [tool.poetry.dependencies]
 python = "^3.10"
-Jinja2 = "3.1.4"
+Jinja2 = "^3.1.5"
 MarkupSafe = "2.1.5"
 beautifulsoup4 = "4.12.2"
 certifi = ">=2024.07.04"

diff --git a/report_examples/html_report_example/01-robots.txt b/report_examples/html_report_example/01-robots.txt
@@ -0,0 +1,3 @@
+User-agent: *
+Disallow: /missions/
+Disallow: /killing/all/humans/