Skip to content

Bigger sota patch 4 #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Bigger sota patch 4 #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
aef0179
'Refactored by Sourcery'
Dec 1, 2023
70498d4
Merge pull request #1 from BiggerSota/sourcery/main
Dec 1, 2023
9f9ba56
Create docker-image.yml
Dec 1, 2023
4e15517
🤖
qb-auto Dec 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/workflows/docker-image.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
name: Docker Image CI

on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]

jobs:

build:

runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- name: Build the Docker image
run: docker build . --file Dockerfile --tag my-image-name:$(date +%s)
100 changes: 53 additions & 47 deletions app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ def get_language_by_guessing(self, text):
with suppress(Exception):
lang = detect(text)
if lang and lang != "":
return self.languages_json[lang] + " (Maybe)"
return f"{self.languages_json[lang]} (Maybe)"
return "unavailable"

def get_language_by_parsing(self, source, encoding):
Expand All @@ -129,14 +129,13 @@ def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
if on_off:
try:
with suppress(Exception):
return func(*args, **kwargs)
except Exception as err:
pass
# if not self.silent: self.log.info(e)
else:
return func(*args, **kwargs)

return wrapper

return decorator

def setup_logger(self, uuid=None, file=False, argv=None):
Expand All @@ -163,39 +162,41 @@ def emit(self, record):
emit, based on user choices
'''

if self.argv.output != "json" and self.sa_object.silent == False:
if isinstance(record.msg, Mapping):
if "custom" in record.msg:
for item in record.msg["custom"]:
with suppress(Exception):
if item == record.msg["custom"][0]:
print("-----------------------")
for key, value in item.items():
if key == "metadata" or key == "extracted":
if (self.argv.metadata and key == "metadata") or (self.argv.extract and key == "extracted"):
with suppress(Exception):
for idx, _item in enumerate(value):
empty_string = key + " " + str(idx)
empty_string = colored(empty_string.ljust(13, ' '), 'blue') + ": "
for _item_key, _item_value in _item.items():
if self.argv.trim and _item_key == "content" and len(_item_value) > 50:
empty_string += "{} : {} ".format(colored(_item_key, 'blue'), colored(_item_value[:50] + "..", 'yellow'))
else:
empty_string += "{} : {} ".format(colored(_item_key, 'blue'), colored(_item_value, 'yellow'))
print("{}".format(empty_string))
else:
print(colored(key.ljust(13, ' '), 'blue'), colored(value, 'yellow'), sep=": ")
if self.argv.output == "json" or self.sa_object.silent != False:
return
if isinstance(record.msg, Mapping):
if "custom" in record.msg:
for item in record.msg["custom"]:
with suppress(Exception):
if item == record.msg["custom"][0]:
print("-----------------------")
else:
print(record.msg)
for key, value in item.items():
if (
key == "metadata"
and self.argv.metadata
or key != "metadata"
and key == "extracted"
and self.argv.extract
):
with suppress(Exception):
for idx, _item in enumerate(value):
empty_string = f"{key} {str(idx)}"
empty_string = colored(empty_string.ljust(13, ' '), 'blue') + ": "
for _item_key, _item_value in _item.items():
if self.argv.trim and _item_key == "content" and len(_item_value) > 50:
empty_string += f"""{colored(_item_key, 'blue')} : {colored(f"{_item_value[:50]}..", 'yellow')} """
else:
empty_string += f"{colored(_item_key, 'blue')} : {colored(_item_value, 'yellow')} "
print(f"{empty_string}")
elif key not in ["metadata", "extracted"]:
print(colored(key.ljust(13, ' '), 'blue'), colored(value, 'yellow'), sep=": ")
print("-----------------------")
else:
print(record.msg)

temp_folder = ''
if argv.logs:
if self.logs_dir != '':
temp_folder = self.logs_dir
else:
temp_folder = mkdtemp()

temp_folder = self.logs_dir if self.logs_dir != '' else mkdtemp()
if file and uuid:
if argv.screenshots:
self.screenshots = True
Expand All @@ -214,7 +215,7 @@ def emit(self, record):

if argv.logs and argv.output != "json":
if not self.silent:
self.log.info('[init] Temporary Logs Directory {}'.format(temp_folder))
self.log.info(f'[init] Temporary Logs Directory {temp_folder}')

def init_detections(self, detections):
'''
Expand Down Expand Up @@ -245,8 +246,7 @@ def search_and_change(self, site, _dict):

def top_websites(self, top_number):
with suppress(Exception):
top_websites = research(self.top_pattern, top_number)
if top_websites:
if top_websites := research(self.top_pattern, top_number):
sites = ([d for d in self.websites_entries if d.get('global_rank') != 0])
sites = sorted(sites, key=lambda x: x['global_rank'])
for site in sites[:int(top_websites.group(1))]:
Expand Down Expand Up @@ -282,7 +282,7 @@ def fetch_url(self, site, username, options):
checking_url = get_fld(site["url"])
checking_url = checking_url.replace(".{username}", "").replace("{username}.", "")
if not self.silent:
self.log.info("[Checking] " + checking_url)
self.log.info(f"[Checking] {checking_url}")

source = ""

Expand Down Expand Up @@ -311,7 +311,7 @@ def fetch_url(self, site, username, options):
source = response.text
content = response.content
encoding = response.encoding
answer = dict((k.lower(), v.lower()) for k, v in response.headers.items())
answer = {k.lower(): v.lower() for k, v in response.headers.items()}
session.close()
temp_profile = {}
temp_detected = {}
Expand All @@ -324,7 +324,7 @@ def check_url(url):

with suppress(Exception):
result = urlparse(url)
if result.scheme == "http" or result.scheme == "https":
if result.scheme in ["http", "https"]:
return all([result.scheme, result.netloc])
return False

Expand Down Expand Up @@ -441,7 +441,7 @@ def detect():
temp_matches.append(parsed)
temp_matches_list.append({"name": item["type"], "value": unquote(match)})

if len(temp_matches_list) > 0:
if temp_matches_list:
temp_profile["extracted"] = temp_matches_list

temp_profile["text"] = temp_profile["text"].replace("\n", "").replace("\t", "").replace("\r", "").strip()
Expand All @@ -466,7 +466,7 @@ def detect():
with suppress(Exception):
if detections_count != 0:
temp_value = round(((temp_profile["found"] / detections_count) * 100), 2)
temp_profile["rate"] = "%" + str(temp_value)
temp_profile["rate"] = f"%{str(temp_value)}"
if temp_value >= 100.00:
temp_profile["status"] = "good"
elif temp_value >= 50.00 and temp_value < 100.00:
Expand All @@ -487,14 +487,20 @@ def detect():
temp_mata_item = {}
add = True
if meta.has_attr("property"):
temp_mata_item.update({"property": meta["property"]})
temp_mata_item["property"] = meta["property"]
if meta.has_attr("content"):
if meta["content"].replace("\n", "").replace("\t", "").replace("\r", "").strip() != "":
temp_mata_item.update({"content": meta["content"].replace("\n", "").replace("\t", "").replace("\r", "").strip()})
temp_mata_item["content"] = (
meta["content"]
.replace("\n", "")
.replace("\t", "")
.replace("\r", "")
.strip()
)
if meta.has_attr("itemprop"):
temp_mata_item.update({"itemprop": meta["itemprop"]})
temp_mata_item["itemprop"] = meta["itemprop"]
if meta.has_attr("name"):
temp_mata_item.update({"name": meta["name"]})
temp_mata_item["name"] = meta["name"]

with suppress(Exception):
if "property" in temp_mata_item:
Expand All @@ -516,7 +522,7 @@ def detect():
temp_meta_list[i]["content"] += ", " + temp_mata_item["content"]
add = False

if len(temp_mata_item) > 0 and add:
if temp_mata_item and add:
temp_meta_list.append(temp_mata_item)

if len(temp_meta_list) > 0:
Expand Down
2 changes: 1 addition & 1 deletion clean-up.logs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
Tue Aug 8 22:26:51 UTC 2023
Fri Dec 1 09:15:48 UTC 2023
[X] social-analyzer
[X] installing autopep8 & jq
[X] running autopep8
Expand Down