web2vec package

Subpackages

• web2vec.crawlers package
  • Submodules
    • web2vec.crawlers.extractors module
      • CertificateExtractor
      • DNSExtractor
      • Extractor
      • GoogleIndexExtractor
      • HtmlBodyExtractor
      • HttpResponseExtractor
      • OpenPageRankExtractor
      • OpenPhishExtractor
      • PhishTankExtractor
      • SimilarWebExtractor
      • UrlGeoExtractor
      • UrlHausExtractor
      • UrlLexicalExtractor
      • WhoisExtractor
      • process_extractors()
    • web2vec.crawlers.models module
      • WebPage
    • web2vec.crawlers.spiders module
      • Web2VecSpider
  • Module contents
• web2vec.extractors package
  • Subpackages
    • web2vec.extractors.external_api package
      • Submodules
      • Module contents
  • Submodules
    • web2vec.extractors.dns_features module
      • DNSFeatures
      • DNSRecordFeatures
      • get_dns_features()
      • get_dns_features_cached()
    • web2vec.extractors.html_body_features module
      • HtmlBodyFeatures
      • body_length()
      • body_to_special_char_ratio()
      • check_obfuscated_scripts()
      • check_suspicious_keywords()
      • detect_api_endpoints()
      • detect_likely_js_spa()
      • find_copyright()
      • find_favicon()
      • find_logo()
      • get_html_body_features()
      • hidden_elements()
      • iframe_redirection()
      • is_external_url()
      • mouse_over_effect()
      • num_email_forms()
      • num_external_iframes()
      • num_external_scripts()
      • num_external_styles()
      • num_forms()
      • num_forms_external_action()
      • num_forms_get()
      • num_forms_post()
      • num_iframes_http()
      • num_images()
      • num_internal_links()
      • num_links()
      • num_media_external()
      • num_media_http()
      • num_meta_tags()
      • num_safe_anchors()
      • num_scripts_http()
      • num_styles_http()
      • num_titles()
      • right_click_disabled()
      • script_length()
      • script_to_body_ratio()
      • script_to_special_chars_ratio()
      • special_characters()
    • web2vec.extractors.http_response_features module
      • HttpResponseFeatures
      • body_length()
      • body_to_special_char_ratio()
      • check_forms()
      • check_header_content_security_policy()
      • check_header_strict_transport_security()
      • check_header_x_content_type_options()
      • check_header_x_frame_options()
      • check_header_x_xss_protection()
      • check_https()
      • check_obfuscated_scripts()
      • check_redirects()
      • check_server_version()
      • check_suspicious_keywords()
      • count_redirects()
      • get_http_response_features()
      • is_live()
      • num_images()
      • num_links()
      • num_titles()
      • script_length()
      • script_to_body_ratio()
      • script_to_special_chars_ratio()
      • special_characters()
    • web2vec.extractors.network_features module
      • build_graph()
    • web2vec.extractors.ssl_certification_features module
      • CertificateFeatures
      • check_ssl()
      • get_certificate_features()
      • get_certificate_features_cached()
      • get_tls_certificate()
      • is_certificate_trusted()
      • is_certificate_valid()
    • web2vec.extractors.url_geo_features module
      • GeoLiteDbType
      • URLGeoFeatures
      • get_asn()
      • get_country()
      • get_geolite_db_files()
      • get_url_geo_features()
      • get_url_geo_features_cached()
    • web2vec.extractors.url_lexical_features module
      • URLLexicalFeatures
      • contains_keywords()
      • count_char()
      • count_vowels()
      • get_url_lexical_features()
      • get_url_lexical_features_cached()
      • has_repeated_digits()
      • numeric_chars_ratio()
      • tld_count()
      • token_count()
      • url_depth()
      • uses_shortening_service()
    • web2vec.extractors.whois_features module
      • WhoisFeatures
      • get_whois_features()
      • get_whois_features_cached()
  • Module contents

Submodules

• web2vec.config module
  • Config
    • Config.api_timeout
    • Config.brave_search_api_key
    • Config.crawler_output_path
    • Config.crawler_spider_depth_limit
    • Config.default_output_path
    • Config.model_config
    • Config.open_page_rank_api_key
    • Config.remote_url_output_path
    • Config.set_correct_path()
    • Config.ssl_verify
• web2vec.utils module
  • create_directories()
  • entropy()
  • fetch_file_from_url()
  • fetch_file_from_url_and_read()
  • fetch_url()
  • get_domain_from_url()
  • get_file_path_for_url()
  • get_github_repo_release_info()
  • get_ip_from_domain()
  • get_ip_from_url()
  • is_numerical_type()
  • sanitize_filename()
  • store_json()
  • transform_value()
  • valid_ip()
• web2vec.version module

Module contents