Skip to content

Commit c4ae681

Browse files
mercamerca
authored
feat: Add script to convert .dot file to graph database (#17)
This commit adds a new Python script `convert.py` that converts a `.dot` file into a graph database. The script uses the `pydot` library to parse the `.dot` file and creates a graph object. It then parses the labels of nodes using regular expressions and updates the traversal code accordingly. The script also loads environment variables using `dotenv`, establishes a connection with Neo4j using the provided URI, username, and password, and uploads the parsed nodes and relationships to the graph database. The commit also includes the addition of a sample `.dot` file in `src/stargripcorp.dataplatform.infra.azure/graph.dot`. Co-authored-by: Merca Ovnerud <[email protected]>
1 parent 316a1c3 commit c4ae681

File tree

2 files changed

+146
-0
lines changed

2 files changed

+146
-0
lines changed

‎scripts/dot_to_graphdb/convert.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
import pydot
2+
import os
3+
import re
4+
from neo4j import GraphDatabase
5+
from dotenv import load_dotenv
6+
7+
# Load the environment variables
8+
load_dotenv()
9+
10+
# Load the .dot file and create a graph
11+
dot_file_path = 'src/stargripcorp.dataplatform.infra.azure/graph.dot'
12+
graphs = pydot.graph_from_dot_file(dot_file_path)
13+
if graphs is None:
14+
raise ValueError('Invalid .dot file')
15+
16+
graph = graphs[0]
17+
18+
# Function to parse the labels
19+
def parse_label(label):
20+
label = label.replace('"', '') # Remove quotes from the label
21+
pattern = re.compile(
22+
r'^urn:pulumi:(?P<stack>[^:]+)::[^:]+::pkg:(?P<pkg>[^:]+):(?P<type>[^$]+)\$(?P<subtype>[^:]+):(?P<name>.+)$'
23+
)
24+
match = pattern.match(label)
25+
if match:
26+
return match.groupdict()
27+
return None # Return None if regex match fails
28+
29+
# Updating the traversal code to parse the labels
30+
nodes, relationships = {}, []
31+
for edge in graph.get_edges():
32+
relationships.append((edge.get_source(), edge.get_destination()))
33+
for node in graph.get_nodes():
34+
node_name = node.get_name()
35+
label = node.get_label() if node.get_label() else node_name # Default to node name if label is absent
36+
parsed_label = parse_label(label)
37+
if parsed_label:
38+
nodes[node_name] = parsed_label
39+
else:
40+
nodes[node_name] = {'name': node_name} # Default to node name if parsing fails
41+
42+
# URI examples: 'neo4j://localhost', 'neo4j+s://xxx.databases.neo4j.io'
43+
URI = os.getenv('NEO4J_URI')
44+
user_name = os.getenv('NEO4J_USER')
45+
password = os.getenv('NEO4J_PASSWORD')
46+
AUTH =(user_name,password)
47+
48+
if URI is None:
49+
raise ValueError('Invalid URI')
50+
51+
52+
53+
def upload_graph(tx, nodes, relationships):
54+
# Create nodes, with parsed label information
55+
for node_name, label_info in nodes.items():
56+
57+
n_name=label_info.get('name', node_name)
58+
n_type=label_info.get('type', 'Unknown')
59+
n_stack=label_info.get('stack','Unknown')
60+
n_pkg=label_info.get('pkg', 'Unknown')
61+
n_subtype=label_info.get('subtype', 'Unknown')
62+
63+
if n_name != node_name:
64+
tx.run(
65+
"""
66+
MERGE (:PulumiNode {
67+
id: $id,
68+
name: $name,
69+
type: $type,
70+
stack: $stack,
71+
pkg: $pkg,
72+
subtype: $subtype
73+
})
74+
""",
75+
id=node_name, # node_name serves as the id
76+
name=n_name,
77+
type=n_type,
78+
stack=n_stack,
79+
pkg=n_pkg,
80+
subtype=n_subtype
81+
)
82+
# Create relationships
83+
for source, destination in relationships:
84+
tx.run(
85+
'''
86+
MATCH (a:PulumiNode {id: $source}), (b:PulumiNode {id: $destination})
87+
MERGE (a)-[:DEPENDS_ON]->(b)
88+
''',
89+
source=source, destination=destination
90+
)
91+
92+
# Invoke the script
93+
with GraphDatabase.driver(URI, auth=AUTH) as driver:
94+
with driver.session() as session:
95+
session.execute_write(upload_graph, nodes, relationships)

‎src/stargripcorp.dataplatform.infra.azure/graph.dot‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
strict digraph {
2+
Resource0 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-data-dev-ra-3a668e53-336b-4d30-94bf-6620cdd036ec"];
3+
Resource0 -> Resource1 [color = "#AA6639"];
4+
Resource2 [label="urn:pulumi:current::stargripcorp::pulumi:pulumi:Stack::stargripcorp-current"];
5+
Resource3 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:consumption:Budget::merca-core-dev-budget"];
6+
Resource3 -> Resource4 [color = "#AA6639"];
7+
Resource5 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Secret::merca-data-dev-secret"];
8+
Resource5 -> Resource1 [color = "#AA6639"];
9+
Resource6 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:resources:ResourceGroup::merca-data-dev-rg"];
10+
Resource6 -> Resource7 [color = "#AA6639"];
11+
Resource6 -> Resource8 [color = "#246C60", label = "resourceGroupName"];
12+
Resource6 -> Resource9 [color = "#246C60", label = "resourceGroupName"];
13+
Resource6 -> Resource5 [color = "#246C60", label = "resourceGroupName"];
14+
Resource1 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault::data-kv"];
15+
Resource1 -> Resource2 [color = "#AA6639"];
16+
Resource10 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault::core-kv"];
17+
Resource10 -> Resource2 [color = "#AA6639"];
18+
Resource11 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:resources:ResourceGroup::merca-core-dev-rg"];
19+
Resource11 -> Resource4 [color = "#AA6639"];
20+
Resource11 -> Resource12 [color = "#246C60", label = "resourceGroupName"];
21+
Resource11 -> Resource13 [color = "#246C60", label = "resourceGroupName"];
22+
Resource14 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-core-dev-ra-3a668e53-336b-4d30-94bf-6620cdd036ec"];
23+
Resource14 -> Resource10 [color = "#AA6639"];
24+
Resource15 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-core-dev-ra-f5d889ea-e64e-467a-9240-f875ff284c04"];
25+
Resource15 -> Resource10 [color = "#AA6639"];
26+
Resource7 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group::data-rg"];
27+
Resource7 -> Resource2 [color = "#AA6639"];
28+
Resource16 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group$azure-native:consumption:Budget::merca-data-dev-budget"];
29+
Resource16 -> Resource7 [color = "#AA6639"];
30+
Resource8 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Vault::merca-data-dev-kv"];
31+
Resource8 -> Resource1 [color = "#AA6639"];
32+
Resource8 -> Resource17 [color = "#246C60", label = "scope"];
33+
Resource8 -> Resource5 [color = "#246C60", label = "vaultName"];
34+
Resource8 -> Resource0 [color = "#246C60", label = "scope"];
35+
Resource17 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:authorization:RoleAssignment::merca-data-dev-ra-f5d889ea-e64e-467a-9240-f875ff284c04"];
36+
Resource17 -> Resource1 [color = "#AA6639"];
37+
Resource4 [label="urn:pulumi:current::stargripcorp::pkg:azure:resource_group::core-rg"];
38+
Resource4 -> Resource2 [color = "#AA6639"];
39+
Resource18 [label="urn:pulumi:current::stargripcorp::pulumi:providers:azure-native::default_2_9_0"];
40+
Resource12 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Vault::merca-core-dev-kv"];
41+
Resource12 -> Resource14 [color = "#246C60", label = "scope"];
42+
Resource12 -> Resource10 [color = "#AA6639"];
43+
Resource12 -> Resource15 [color = "#246C60", label = "scope"];
44+
Resource12 -> Resource13 [color = "#246C60", label = "vaultName"];
45+
Resource13 [label="urn:pulumi:current::stargripcorp::pkg:azure:keyvault$azure-native:keyvault:Secret::merca-core-dev-secret"];
46+
Resource13 -> Resource10 [color = "#AA6639"];
47+
Resource19 [label="urn:pulumi:current::stargripcorp::pkg:azure:storage::data-sa"];
48+
Resource19 -> Resource2 [color = "#AA6639"];
49+
Resource9 [label="urn:pulumi:current::stargripcorp::pkg:azure:storage$azure-native:storage:StorageAccount::merca-data-dev-sa"];
50+
Resource9 -> Resource19 [color = "#AA6639"];
51+
}

0 commit comments

Comments
 (0)