Jun 13, 2023, 10:40 AM
=================================================================================
import requests
from bs4 import BeautifulSoup
import nltk
import nltk.sentiment.vader as vader
from pygments.lexers import HtmlLexer, CssLexer, JavascriptLexer
from pygments import lex
import difflib
import time
import re
import urllib.parse
import ssl
import socket
def main():
while True:
print('\nSelect an option:')
print('1. Calculate keyword density')
print('2. Analyze sentiment')
print('3. Analyze links')
print('4. Validate code')
print('5. Check for duplicate content')
print('6. Analyze search engine optimization')
print('7. Analyze security')
print('8. Analyze website architecture')
print('9. Analyze page load speed')
print('10. Analyze mobile responsiveness')
print('0. Exit')
option = input('Option: ')
if option == '1':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
keywords = input('Enter the keywords you want to check the density for (comma-separated): ').split(',')
keyword_density = calculate_keyword_density(response.content, keywords)
print('Keyword density:', keyword_density)
elif option == '2':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
sentiment = analyze_sentiment(response.content)
print('Sentiment:', sentiment)
elif option == '3':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
links = analyze_links(response.content)
print('Links:', links)
elif option == '4':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
errors = validate_code(response.content)
print('Code validation errors:', errors)
elif option == '5':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
duplicates = check_for_duplicates(response.content)
print('Duplicate content:', duplicates)
elif option == '6':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
seo = analyze_seo(response.content)
print('SEO analysis:', seo)
elif option == '7':
url = input('Enter the URL of the website you want to analyze: ')
security = analyze_security(url)
print('Security analysis:', security)
elif option == '8':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
architecture = analyze_architecture(response.content)
print('Website architecture analysis:', architecture)
elif option == '9':
url = input('Enter the URL of the website you want to analyze: ')
page_load_time = analyze_page_load_time(url)
print('Page load time:', page_load_time)
elif option == '10':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
mobile_responsiveness = analyze_mobile_responsiveness(response.content)
print('Mobile responsiveness analysis:', mobile_responsiveness)
elif option == '0':
break
else:
print('Invalid option selected')
print('Exiting program...')
def calculate_keyword_density(content, keywords):
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
tokens = nltk.word_tokenize(text)
frequency = nltk.FreqDist(tokens)
total_words = sum(frequency.values())
keyword_count = sum(frequency[keyword] for keyword in keywords)
return keyword_count / total_words * 100
def analyze_sentiment(content):
analyzer = vader.SentimentIntensityAnalyzer()
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
sentiment_scores = analyzer.polarity_scores(text)
return sentiment_scores
def analyze_links(content):
soup = BeautifulSoup(content, 'html.parser')
links = [link.get('href') for link in soup.find_all('a')]
return links
def validate_code(content):
errors = []
for lexer in [HtmlLexer, CssLexer, JavascriptLexer]:
code = ''.join(lex(content, lexer()))
checker = difflib.Differ()
diff = checker.compare(code.splitlines(), content.splitlines())
diff = '\n'.join(diff)
if '- ' in diff or '+ ' in diff:
errors.append(diff)
return errors
def check_for_duplicates(content):
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
tokens = nltk.word_tokenize(text)
frequency = nltk.FreqDist(tokens)
duplicates = []
for word, count in frequency.items():
if count > 1 and len(word) > 3:
duplicates.append(word)
return duplicates
def analyze_seo(content):
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
word_count = len(text.split())
title = soup.title.string if soup.title else ''
description = soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else ''
h1_count = len(soup.find_all('h1'))
h2_count = len(soup.find_all('h2'))
h3_count = len(soup.find_all('h3'))
img_count = len(soup.find_all('img'))
internal_links_count = len(soup.find_all('a', href=re.compile('^(/|' + urllib.parse.urlparse(requests.get(content).url).netloc + ')')))
external_links_count = len(soup.find_all('a', href=re.compile('^(http|https)://')))
return {
'word_count': word_count,
'title': title,
'description': description,
'h1_count': h1_count,
'h2_count': h2_count,
'h3_count': h3_count,
'img_count': img_count,
'internal_links_count': internal_links_count,
'external_links_count': external_links_count
}
def analyze_security(url):
try:
requests.get(url, verify=True)
except (ssl.CertificateError, ssl.SSLError, socket.timeout):
return 'Not secure'
return 'Secure'
def analyze_architecture(content):
soup = BeautifulSoup(content, 'html.parser')
scripts = len(soup.find_all('script'))
stylesheets = len(soup.find_all('link', {'rel': 'stylesheet'}))
images = len(soup.find_all('img'))
videos = len(soup.find_all('video'))
audio = len(soup.find_all('audio'))
forms = len(soup.find_all('form'))
iframes = len(soup.find_all('iframe'))
return {
'scripts': scripts,
'stylesheets': stylesheets,
'images': images,
'videos': videos,
'audio': audio,
'forms': forms,
'iframes': iframes
}
def analyze_page_load_time(url):
start_time = time.time()
requests.get(url)
end_time = time.time()
return end_time - start_time
def analyze_mobile_responsiveness(content):
soup = BeautifulSoup(content, 'html.parser')
viewport = soup.find('meta', {'name': 'viewport'})
if viewport:
return 'Mobile friendly'
return 'Not mobile friendly'
if __name__ == '__main__':
main()
========================================
import requests
from bs4 import BeautifulSoup
import nltk
import nltk.sentiment.vader as vader
from pygments.lexers import HtmlLexer, CssLexer, JavascriptLexer
from pygments import lex
import difflib
import time
import re
import urllib.parse
import ssl
import socket
def main():
while True:
print('\nSelect an option:')
print('1. Calculate keyword density')
print('2. Analyze sentiment')
print('3. Analyze links')
print('4. Validate code')
print('5. Check for duplicate content')
print('6. Analyze search engine optimization')
print('7. Analyze security')
print('8. Analyze website architecture')
print('9. Analyze page load speed')
print('10. Analyze mobile responsiveness')
print('0. Exit')
option = input('Option: ')
if option == '1':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
keywords = input('Enter the keywords you want to check the density for (comma-separated): ').split(',')
keyword_density = calculate_keyword_density(response.content, keywords)
print('Keyword density:', keyword_density)
elif option == '2':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
sentiment = analyze_sentiment(response.content)
print('Sentiment:', sentiment)
elif option == '3':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
links = analyze_links(response.content)
print('Links:', links)
elif option == '4':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
errors = validate_code(response.content)
print('Code validation errors:', errors)
elif option == '5':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
duplicates = check_for_duplicates(response.content)
print('Duplicate content:', duplicates)
elif option == '6':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
seo = analyze_seo(response.content)
print('SEO analysis:', seo)
elif option == '7':
url = input('Enter the URL of the website you want to analyze: ')
security = analyze_security(url)
print('Security analysis:', security)
elif option == '8':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
architecture = analyze_architecture(response.content)
print('Website architecture analysis:', architecture)
elif option == '9':
url = input('Enter the URL of the website you want to analyze: ')
page_load_time = analyze_page_load_time(url)
print('Page load time:', page_load_time)
elif option == '10':
url = input('Enter the URL of the website you want to analyze: ')
response = requests.get(url)
mobile_responsiveness = analyze_mobile_responsiveness(response.content)
print('Mobile responsiveness analysis:', mobile_responsiveness)
elif option == '0':
break
else:
print('Invalid option selected')
print('Exiting program...')
def calculate_keyword_density(content, keywords):
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
tokens = nltk.word_tokenize(text)
frequency = nltk.FreqDist(tokens)
total_words = sum(frequency.values())
keyword_count = sum(frequency[keyword] for keyword in keywords)
return keyword_count / total_words * 100
def analyze_sentiment(content):
analyzer = vader.SentimentIntensityAnalyzer()
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
sentiment_scores = analyzer.polarity_scores(text)
return sentiment_scores
def analyze_links(content):
soup = BeautifulSoup(content, 'html.parser')
links = [link.get('href') for link in soup.find_all('a')]
return links
def validate_code(content):
errors = []
for lexer in [HtmlLexer, CssLexer, JavascriptLexer]:
code = ''.join(lex(content, lexer()))
checker = difflib.Differ()
diff = checker.compare(code.splitlines(), content.splitlines())
diff = '\n'.join(diff)
if '- ' in diff or '+ ' in diff:
errors.append(diff)
return errors
def check_for_duplicates(content):
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
tokens = nltk.word_tokenize(text)
frequency = nltk.FreqDist(tokens)
duplicates = []
for word, count in frequency.items():
if count > 1 and len(word) > 3:
duplicates.append(word)
return duplicates
def analyze_seo(content):
soup = BeautifulSoup(content, 'html.parser')
text = soup.get_text()
word_count = len(text.split())
title = soup.title.string if soup.title else ''
description = soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else ''
h1_count = len(soup.find_all('h1'))
h2_count = len(soup.find_all('h2'))
h3_count = len(soup.find_all('h3'))
img_count = len(soup.find_all('img'))
internal_links_count = len(soup.find_all('a', href=re.compile('^(/|' + urllib.parse.urlparse(requests.get(content).url).netloc + ')')))
external_links_count = len(soup.find_all('a', href=re.compile('^(http|https)://')))
return {
'word_count': word_count,
'title': title,
'description': description,
'h1_count': h1_count,
'h2_count': h2_count,
'h3_count': h3_count,
'img_count': img_count,
'internal_links_count': internal_links_count,
'external_links_count': external_links_count
}
def analyze_security(url):
try:
requests.get(url, verify=True)
except (ssl.CertificateError, ssl.SSLError, socket.timeout):
return 'Not secure'
return 'Secure'
def analyze_architecture(content):
soup = BeautifulSoup(content, 'html.parser')
scripts = len(soup.find_all('script'))
stylesheets = len(soup.find_all('link', {'rel': 'stylesheet'}))
images = len(soup.find_all('img'))
videos = len(soup.find_all('video'))
audio = len(soup.find_all('audio'))
forms = len(soup.find_all('form'))
iframes = len(soup.find_all('iframe'))
return {
'scripts': scripts,
'stylesheets': stylesheets,
'images': images,
'videos': videos,
'audio': audio,
'forms': forms,
'iframes': iframes
}
def analyze_page_load_time(url):
start_time = time.time()
requests.get(url)
end_time = time.time()
return end_time - start_time
def analyze_mobile_responsiveness(content):
soup = BeautifulSoup(content, 'html.parser')
viewport = soup.find('meta', {'name': 'viewport'})
if viewport:
return 'Mobile friendly'
return 'Not mobile friendly'
if __name__ == '__main__':
main()
========================================
