identify_similarities.py

from pyspark.sql import SparkSession
from pyspark import SparkContext, SparkConf
import matplotlib.pyplot as plt 
from pyspark.sql import functions as F

import ssh.shingle as sh
import data_handler
import ssh.hashing as mh


def similarity_names(process_df):
    # Aggregate the server names
    result = data_handler.aggregate_server_names(process_df)
    # Compare server names using minhashing for each number of servers
    first = True
    for i in range(1,16):
        data = result.where(result.number_servers == i)
        if data.count() > 0:
            data =  sh.one_hot(data, "server_names", "one_hot")
            distances_server_name = mh.minhashing(data, "one_hot", threshold=0.4)
            if first:
                distances = distances_server_name
                first = False
            distances = distances.union(distances_server_name)
    return distances


# Collect similar processes
def similar_processes_names(process_df):
    distances = similarity_names(process_df)
    # Select the processes that are similar and remove duplicates
    similar_processes = distances.where(distances.idA < distances.idB).distinct().select("idA", "idB")
    return similar_processes