Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 74 additions & 76 deletions nodestream_github/github_organizations.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,83 +6,81 @@
- implementation: nodestream.interpreting:Interpreter
arguments:
interpretations:
- type: source_node
node_type: GithubOrg
key_normalization:
do_lowercase_strings: false
key:
node_id: !jmespath 'node_id'
additional_indexes:
- name
- login
- html_url
- type: properties
properties:
login: !jmespath 'login'
id: !jmespath 'id'
url: !jmespath 'url'
description: !jmespath 'description'
name: !jmespath 'name'
company: !jmespath 'company'
blog: !jmespath 'blog'
location: !jmespath 'location'
email: !jmespath 'email'
twitter_username: !jmespath 'twitter_username'
is_verified: !jmespath 'is_verified'
has_organization_projects: !jmespath 'has_organization_projects'
has_repository_projects: !jmespath 'has_repository_projects'
public_repos: !jmespath 'public_repos'
public_gists: !jmespath 'public_gists'
followers: !jmespath 'followers'
following: !jmespath 'following'
html_url: !jmespath 'html_url'
created_at: !jmespath 'created_at'
type: !jmespath 'type'
total_private_repos: !jmespath 'total_private_repos'
owned_private_repos: !jmespath 'owned_private_repos'
private_gists: !jmespath 'private_gists'
disk_usage: !jmespath 'disk_usage'
collaborators: !jmespath 'collaborators'
billing_email: !jmespath 'billing_email'
plan: !jmespath 'plan.name'
default_repository_permission: !jmespath 'default_repository_permission'
members_can_create_repositories: !jmespath 'members_can_create_repositories'
two_factor_requirement_enabled: !jmespath 'two_factor_requirement_enabled'
members_allowed_repository_creation_type: !jmespath 'members_allowed_repository_creation_type'
members_can_create_public_repositories: !jmespath 'members_can_create_public_repositories'
members_can_create_private_repositories: !jmespath 'members_can_create_private_repositories'
members_can_create_internal_repositories: !jmespath 'members_can_create_internal_repositories'
members_can_create_pages: !jmespath 'members_can_create_pages'
members_can_create_public_pages: !jmespath 'members_can_create_public_pages'
members_can_create_private_pages: !jmespath 'members_can_create_private_pages'
members_can_fork_private_repositories: !jmespath 'members_can_fork_private_repositories'
web_commit_signoff_required: !jmespath 'web_commit_signoff_required'
updated_at: !jmespath 'updated_at'
dependency_graph_enabled_for_new_repositories: !jmespath 'dependency_graph_enabled_for_new_repositories'
dependabot_alerts_enabled_for_new_repositories: !jmespath 'dependabot_alerts_enabled_for_new_repositories'
dependabot_security_updates_enabled_for_new_repositories: !jmespath 'dependabot_security_updates_enabled_for_new_repositories'
advanced_security_enabled_for_new_repositories: !jmespath 'advanced_security_enabled_for_new_repositories'
secret_scanning_enabled_for_new_repositories: !jmespath 'secret_scanning_enabled_for_new_repositories'
secret_scanning_push_protection_enabled_for_new_repositories: !jmespath 'secret_scanning_push_protection_enabled_for_new_repositories'
secret_scanning_push_protection_custom_link: !jmespath 'secret_scanning_push_protection_custom_link'
secret_scanning_push_protection_custom_link_enabled: !jmespath 'secret_scanning_push_protection_custom_link_enabled'
- type: source_node
node_type: GithubOrg
key_normalization:
do_lowercase_strings: false
key:
node_id: !jmespath 'node_id'
additional_indexes:
- name
- login
- html_url
- type: properties
properties:
login: !jmespath 'login'
id: !jmespath 'id'
url: !jmespath 'url'
description: !jmespath 'description'
name: !jmespath 'name'
company: !jmespath 'company'
blog: !jmespath 'blog'
location: !jmespath 'location'
email: !jmespath 'email'
twitter_username: !jmespath 'twitter_username'
is_verified: !jmespath 'is_verified'
has_organization_projects: !jmespath 'has_organization_projects'
has_repository_projects: !jmespath 'has_repository_projects'
public_repos: !jmespath 'public_repos'
public_gists: !jmespath 'public_gists'
followers: !jmespath 'followers'
following: !jmespath 'following'
html_url: !jmespath 'html_url'
created_at: !jmespath 'created_at'
type: !jmespath 'type'
total_private_repos: !jmespath 'total_private_repos'
owned_private_repos: !jmespath 'owned_private_repos'
private_gists: !jmespath 'private_gists'
disk_usage: !jmespath 'disk_usage'
collaborators: !jmespath 'collaborators'
billing_email: !jmespath 'billing_email'
plan: !jmespath 'plan.name'
default_repository_permission: !jmespath 'default_repository_permission'
members_can_create_repositories: !jmespath 'members_can_create_repositories'
two_factor_requirement_enabled: !jmespath 'two_factor_requirement_enabled'
members_allowed_repository_creation_type: !jmespath 'members_allowed_repository_creation_type'
members_can_create_public_repositories: !jmespath 'members_can_create_public_repositories'
members_can_create_private_repositories: !jmespath 'members_can_create_private_repositories'
members_can_create_internal_repositories: !jmespath 'members_can_create_internal_repositories'
members_can_create_pages: !jmespath 'members_can_create_pages'
members_can_create_public_pages: !jmespath 'members_can_create_public_pages'
members_can_create_private_pages: !jmespath 'members_can_create_private_pages'
members_can_fork_private_repositories: !jmespath 'members_can_fork_private_repositories'
web_commit_signoff_required: !jmespath 'web_commit_signoff_required'
updated_at: !jmespath 'updated_at'
dependency_graph_enabled_for_new_repositories: !jmespath 'dependency_graph_enabled_for_new_repositories'
dependabot_alerts_enabled_for_new_repositories: !jmespath 'dependabot_alerts_enabled_for_new_repositories'
dependabot_security_updates_enabled_for_new_repositories: !jmespath 'dependabot_security_updates_enabled_for_new_repositories'
advanced_security_enabled_for_new_repositories: !jmespath 'advanced_security_enabled_for_new_repositories'
secret_scanning_enabled_for_new_repositories: !jmespath 'secret_scanning_enabled_for_new_repositories'
secret_scanning_push_protection_enabled_for_new_repositories: !jmespath 'secret_scanning_push_protection_enabled_for_new_repositories'
secret_scanning_push_protection_custom_link: !jmespath 'secret_scanning_push_protection_custom_link'
secret_scanning_push_protection_custom_link_enabled: !jmespath 'secret_scanning_push_protection_custom_link_enabled'

- type: github-user-relationship
relationship_type: IS_MEMBER
iterate_on: !jmespath 'members[*]'
outbound: false
key_normalization:
do_lowercase_strings: false
relationship_properties:
role: !jmespath 'role'
- type: github-user-relationship
relationship_type: IS_MEMBER
iterate_on: !jmespath 'members[*]'
outbound: false
key_normalization:
do_lowercase_strings: false
relationship_properties:
role: !jmespath 'role'

- type: github-repo-relationship
relationship_type: IN_ORGANIZATION
iterate_on: !jmespath 'repositories[*]'
outbound: false
key_normalization:
do_lowercase_strings: false
relationship_properties:
permission: !jmespath 'permission'
- type: github-repo-relationship
relationship_type: IN_ORGANIZATION
iterate_on: !jmespath 'repositories[*]'
outbound: false
key_normalization:
do_lowercase_strings: false

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we remove the permission in the IN_ORGANIZATION relationship, where would we store it in the next iteration? Will that be done through the simplify_repo/simplify_user changes you made in /relationship/repository.py and /relationship/user.py?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's in org.default_repository_permissions which is where I've been writing queries from. I don't think we're even writing this permission to our data since our yaml is a custom version of this.


220 changes: 110 additions & 110 deletions nodestream_github/github_repos.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,120 +7,120 @@
- implementation: nodestream.interpreting:Interpreter
arguments:
interpretations:
- type: source_node
node_type: GithubRepo
additional_indexes:
- full_name
- html_url
key_normalization:
do_lowercase_strings: false
key:
node_id: !jmespath 'node_id'
- type: properties
properties:
id: !jmespath 'id'
name: !jmespath 'name'
full_name: !jmespath 'full_name'
private: !jmespath 'private'
html_url: !jmespath 'html_url'
description: !jmespath 'description'
fork: !jmespath 'fork'
url: !jmespath 'url'
homepage: !jmespath 'homepage'
language: !jmespath 'language'
forks_count: !jmespath 'forks_count'
stargazers_count: !jmespath 'stargazers_count'
watchers_count: !jmespath 'watchers_count'
size: !jmespath 'size'
default_branch: !jmespath 'default_branch'
open_issues_count: !jmespath 'open_issues_count'
is_template: !jmespath 'is_template'
topics: !jmespath 'topics'
has_issues: !jmespath 'has_issues'
has_projects: !jmespath 'has_projects'
has_wiki: !jmespath 'has_wiki'
has_pages: !jmespath 'has_pages'
has_downloads: !jmespath 'has_downloads'
has_discussions: !jmespath 'has_discussions'
archived: !jmespath 'archived'
disabled: !jmespath 'disabled'
visibility: !jmespath 'visibility'
pushed_at: !jmespath 'pushed_at'
created_at: !jmespath 'created_at'
updated_at: !jmespath 'updated_at'
security_and_analysis_advanced_security: !jmespath 'security_and_analysis.advanced_security.status'
security_and_analysis_secret_scanning: !jmespath 'security_and_analysis.secret_scanning.status'
security_and_analysis_secret_scanning_push_protection: !jmespath 'security_and_analysis.secret_scanning_push_protection.status'
- type: source_node
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

curse ye, autoformat

node_type: GithubRepo
additional_indexes:
- full_name
- html_url
key_normalization:
do_lowercase_strings: false
key:
node_id: !jmespath 'node_id'
- type: properties
properties:
id: !jmespath 'id'
name: !jmespath 'name'
full_name: !jmespath 'full_name'
private: !jmespath 'private'
html_url: !jmespath 'html_url'
description: !jmespath 'description'
fork: !jmespath 'fork'
url: !jmespath 'url'
homepage: !jmespath 'homepage'
language: !jmespath 'language'
forks_count: !jmespath 'forks_count'
stargazers_count: !jmespath 'stargazers_count'
watchers_count: !jmespath 'watchers_count'
size: !jmespath 'size'
default_branch: !jmespath 'default_branch'
open_issues_count: !jmespath 'open_issues_count'
is_template: !jmespath 'is_template'
topics: !jmespath 'topics'
has_issues: !jmespath 'has_issues'
has_projects: !jmespath 'has_projects'
has_wiki: !jmespath 'has_wiki'
has_pages: !jmespath 'has_pages'
has_downloads: !jmespath 'has_downloads'
has_discussions: !jmespath 'has_discussions'
archived: !jmespath 'archived'
disabled: !jmespath 'disabled'
visibility: !jmespath 'visibility'
pushed_at: !jmespath 'pushed_at'
created_at: !jmespath 'created_at'
updated_at: !jmespath 'updated_at'
security_and_analysis_advanced_security: !jmespath 'security_and_analysis.advanced_security.status'
security_and_analysis_secret_scanning: !jmespath 'security_and_analysis.secret_scanning.status'
security_and_analysis_secret_scanning_push_protection: !jmespath 'security_and_analysis.secret_scanning_push_protection.status'

- type: relationship
node_type: GithubOrg
relationship_type: IN_ORGANIZATION
key_normalization:
do_lowercase_strings: false
node_key:
node_id: !jmespath 'org_owner.node_id'
node_properties:
id: !jmespath "org_owner.id"
name: !jmespath "org_owner.name"
login: !jmespath "org_owner.login"
- type: relationship
node_type: GithubOrg
relationship_type: IN_ORGANIZATION
key_normalization:
do_lowercase_strings: false
node_key:
node_id: !jmespath 'org_owner.node_id'
node_properties:
id: !jmespath "org_owner.id"
name: !jmespath "org_owner.name"
login: !jmespath "org_owner.login"


- type: relationship
node_type: GithubUser
relationship_type: IS_OWNER
outbound: false
key_normalization:
do_lowercase_strings: false
node_key:
node_id: !jmespath 'user_owner.node_id'
node_properties:
id: !jmespath "user_owner.id"
login: !jmespath "user_owner.login"
- type: relationship
node_type: GithubUser
relationship_type: IS_OWNER
outbound: false
key_normalization:
do_lowercase_strings: false
node_key:
node_id: !jmespath 'user_owner.node_id'
node_properties:
id: !jmespath "user_owner.id"
login: !jmespath "user_owner.login"

- type: github-user-relationship
relationship_type: IS_COLLABORATOR
iterate_on: !jmespath 'collaborators[*]'
outbound: false
key_normalization:
do_lowercase_strings: false
relationship_properties:
"affiliation": !jmespath "affiliation"
"permission": !jmespath "role_name"
- type: github-user-relationship
relationship_type: IS_COLLABORATOR
iterate_on: !jmespath 'collaborators[*]'
outbound: false
key_normalization:
do_lowercase_strings: false
relationship_properties:
"affiliation": !jmespath "affiliation"
"role_name": !jmespath "role_name"

- type: relationship
node_type: ProgrammingLanguage
relationship_type: HAS_LANGUAGE
key_normalization:
do_lowercase_strings: true
node_key:
name: !jmespath 'name'
relationship_properties:
size: !jmespath size
iterate_on: !jmespath 'languages'
- type: relationship
node_type: ProgrammingLanguage
relationship_type: HAS_LANGUAGE
key_normalization:
do_lowercase_strings: true
node_key:
name: !jmespath 'name'
relationship_properties:
size: !jmespath size
iterate_on: !jmespath 'languages'

- type: relationship
node_type: GithubWebhook
iterate_on: !jmespath 'webhooks[*]'
relationship_type: WEBHOOK_ON
outbound: false
key_normalization:
do_lowercase_strings: false
relationship_properties:
events: !jmespath "events"
node_key:
id: !jmespath 'id'
node_properties:
type: !jmespath 'type'
id: !jmespath 'id'
name: !jmespath 'name'
active: !jmespath 'active'
config_content_type: !jmespath 'config.content_type'
config_insecure_ssl: !jmespath 'config.insecure_ssl'
config_url: !jmespath 'config.url'
updated_at: !jmespath 'updated_at'
created_at: !jmespath 'created_at'
url: !jmespath 'url'
last_response_code: !jmespath 'last_response.code'
last_response_status: !jmespath 'last_response.status'
last_response_message: !jmespath 'last_response.message'
- type: relationship
node_type: GithubWebhook
iterate_on: !jmespath 'webhooks[*]'
relationship_type: WEBHOOK_ON
outbound: false
key_normalization:
do_lowercase_strings: false
relationship_properties:
events: !jmespath "events"
node_key:
id: !jmespath 'id'
node_properties:
type: !jmespath 'type'
id: !jmespath 'id'
name: !jmespath 'name'
active: !jmespath 'active'
config_content_type: !jmespath 'config.content_type'
config_insecure_ssl: !jmespath 'config.insecure_ssl'
config_url: !jmespath 'config.url'
updated_at: !jmespath 'updated_at'
created_at: !jmespath 'created_at'
url: !jmespath 'url'
last_response_code: !jmespath 'last_response.code'
last_response_status: !jmespath 'last_response.status'
last_response_message: !jmespath 'last_response.message'

Loading