Merge pull request #241 from rackerlabs/issue-238

Console/Engine updates

Author: derpadoo

README.md

@@ -332,10 +332,13 @@ virtualenv -p python3.6 .venv
 source .venv/bin/activate
 pip install pyinstaller
 pyinstaller --onefile engine.py --name engine
+mv dist/engine .
 
 ./engine -v > standalone_engine_binary.txt
 echo -e "MD5     `md5sum engine`" >> standalone_engine_binary.txt
 echo -e "SHA-256 `sha256sum engine`" >> standalone_engine_binary.txt
+
+rm -rf __pycache__ build dist engine.spec .venv
 ```
 
 ### Engine Execution

ansible-playbooks/roles/console/tasks/main.yml

@@ -408,6 +408,7 @@
     owner: nobody
     group: nogroup
     recurse: yes
+    mode: 0777  # Not ideal, but only way to allow Django's www-data user to move files within scan_results.
   tags: update_code
 
 - name: Restart NFS service.
@@ -435,18 +436,17 @@
     group: root
     mode: 0755
 
-- name: Make nmap_to_csv.sh and nmap_to_csv.py executable.
+- name: Change Python scripts owner and file permissions.
   file:
     path: "/home/{{ non_root_user }}/console/scan_results/{{ item }}"
-    owner: root
-    group: root
-    mode: 0700
+    owner: "{{ non_root_user }}"
+    group: "{{ non_root_user }}"
+    mode: 0755
   with_items:
-    - nmap_to_csv.sh
     - nmap_to_csv.py
-    - masscan_json_to_csv.sh
     - masscan_json_to_csv.py
     - xml_to_json_nmap_results.py
+  tags: update_code
 
 - name: chown-ing console files to root in "/home/{{ non_root_user }}/console"
   file:
@@ -476,30 +476,6 @@
     job: "{{ scantron_dir }}/scan_scheduler.sh"
     user: root
 
-- name: Add crontab entry for nmap_to_csv.sh
-  cron:
-    name: Convert nmap scan files for big data analytics platform ingestion every minute.
-    disabled: false
-    minute: "*"
-    hour: "*"
-    day: "*"
-    month: "*"
-    weekday: "*"
-    job: "{{ scantron_dir }}/scan_results/nmap_to_csv.sh"
-    user: root
-
-- name: Add crontab entry for masscan_json_to_csv.sh
-  cron:
-    name: Convert masscan json scan files for big data analytics platform ingestion every minute.
-    disabled: false
-    minute: "*"
-    hour: "*"
-    day: "*"
-    month: "*"
-    weekday: "*"
-    job: "{{ scantron_dir }}/scan_results/masscan_json_to_csv.sh"
-    user: root
-
 - name: Disable MAILTO for root's crontab.
   cronvar:
     name: MAILTO

console/django_scantron/__init__.py

@@ -1 +1 @@
-__version__ = "1.38"
+__version__ = "1.39"

console/django_scantron/api/views.py

@@ -1,5 +1,6 @@
 # Standard Python libraries.
 import datetime
+import os
 import pytz
 
 # Third party Python libraries.
@@ -163,8 +164,33 @@ def partial_update(self, request, pk=None, **kwargs):
 
                     return JsonResponse(response_dict)
 
-                # Update the scheduled_scan_dict with the most recent scan_status state from the PUT request.  When
-                # originally querying above, the old state is passed to utility.py unless it is updated.
+                # Setup folder directories.
+                scan_results_dir = "/home/scantron/console/scan_results"
+                pending_files_dir = os.path.join(scan_results_dir, "pending")
+                completed_files_dir = os.path.join(scan_results_dir, "complete")
+                cancelled_files_dir = os.path.join(scan_results_dir, "cancelled")
+
+                if new_scan_status == "cancelled":
+                    # Move scan files to the "cancelled" directory for historical purposes.
+                    utility.move_wildcard_files(
+                        f"{scheduled_scan_dict['result_file_base_name']}*", pending_files_dir, cancelled_files_dir
+                    )
+
+                if new_scan_status == "completed":
+                    # Move files from "pending" directory to "complete" directory.
+                    utility.move_wildcard_files(
+                        f"{scheduled_scan_dict['result_file_base_name']}*", pending_files_dir, completed_files_dir
+                    )
+
+                    # Django compliant pre-formated datetimestamp.
+                    now_datetime = get_current_time()
+                    ScheduledScan.objects.filter(scan_engine=request.user).filter(pk=pk).update(
+                        completed_time=now_datetime
+                    )
+
+                # Update the scheduled_scan_dict with the most recent scan_status state from the PATCH request.  When
+                # originally querying above, the old state would passed to utility.py since it hasn't officially been
+                # updated by Django's .update() yet.
                 scheduled_scan_dict["scan_status"] = new_scan_status
 
                 # Create a redis connection object.

console/django_scantron/templates/base.html

@@ -303,7 +303,7 @@ <h1>
             "ordering": true,
             "info": true,
             "autoWidth": true,
-            "order": [[ 6, "desc"]],
+            "order": [[ 0, "desc"]],
             "pageLength": 100
         });
     });

console/django_scantron/templates/django_scantron/scheduled_scan_list.html

@@ -22,7 +22,7 @@ <h3>
             <th>Scan Command</th>
             <th>Scan Status</th>
             <th>Completion Time</th>
-            <th>Results File</th>
+            <th>Result Files</th>
         </tr>
     </thead>
 

console/scan_results/masscan_json_to_csv.sh

@@ -2,7 +2,10 @@
 Utility methods for other scripts to use.
 """
 # Standard Python libraries.
+import fnmatch
 import logging
+import os
+import shutil
 from logging import handlers
 
 # Third party Python libraries.
@@ -11,7 +14,7 @@
 
 # Custom Python libraries.
 import django_connector
-
+from scan_results import nmap_to_csv, masscan_json_to_csv
 
 # Setup logging configuration.
 logger = logging.getLogger("rq.worker")
@@ -37,7 +40,15 @@
 )
 # fmt: on
 
-# https://github.com/pennersr/django-allauth/blob/7b81531bc89ae98dc6f687611743db5b36cda9a2/allauth/account/adapter.py#L448
+
+def move_wildcard_files(wildcard_filename, source_directory, destination_directory):
+    """Move files with supported fnmatch patterns (* and ?)."""
+
+    file_list = os.listdir(source_directory)
+
+    for file_name in file_list:
+        if fnmatch.fnmatch(file_name, wildcard_filename):
+            shutil.move(os.path.join(source_directory, file_name), os.path.join(destination_directory, file_name))
 
 
 def process_scan_status_change(scheduled_scan_dict):
@@ -90,5 +101,11 @@ def process_scan_status_change(scheduled_scan_dict):
 
         logger.info(f"Successfully sent email for Scheduled Scan ID: {scheduled_scan_id}")
 
-    # 2 Do other stuff
-    # TODO
+    # 2) Convert scan results to .csv for big data analytics.
+    # Calling the functions here instead of relying on cron job that runs every minute.  The scripts also moves the
+    # .xml files from console/scan_results/complete to console/scan_results/processed
+    if scan_status == "completed":
+        if scan_binary == "nmap":
+            nmap_to_csv.main()
+        elif scan_binary == "masscan":
+            masscan_json_to_csv.main()