"""
Custom HTML Parser for HTTP Tool.
Provides HTML parsing and CSS selector extraction using Python's stdlib html.parser,
replacing the bs4 (BeautifulSoup4) dependency.
Supports:
- Basic CSS selectors: tag, .class, #id, tag.class, tag#id
- Text extraction with tag stripping
- Element selection for response optimization
"""
import re
from html.parser import HTMLParser
from typing import List, Optional, Tuple
[docs]
class HTMLNode:
"""Represents an HTML element node."""
__slots__ = ('tag', 'attrs', 'children', 'text', 'parent')
def __init__(self, tag: str, attrs: List[Tuple[str, Optional[str]]] = None):
self.tag = tag
self.attrs = dict(attrs) if attrs else {}
self.children: List['HTMLNode'] = []
self.text: str = ""
self.parent: Optional['HTMLNode'] = None
[docs]
def get_classes(self) -> List[str]:
"""Get list of classes from class attribute."""
class_attr = self.attrs.get('class', '')
return class_attr.split() if class_attr else []
[docs]
def get_id(self) -> Optional[str]:
"""Get id attribute value."""
return self.attrs.get('id')
[docs]
def matches_selector(self, selector: str) -> bool:
"""
Check if this node matches a CSS selector.
Supports:
- tag: matches tag name
- .class: matches class
- #id: matches id
- tag.class: matches tag with class
- tag#id: matches tag with id
- .class1.class2: matches multiple classes
"""
if not selector:
return True
# Parse selector into components
# Match patterns like: tag, .class, #id, tag.class, tag#id, .class1.class2
tag_match = re.match(r'^([a-zA-Z][a-zA-Z0-9]*)?', selector)
tag_name = tag_match.group(1) if tag_match else None
# Extract classes (everything after .)
classes = re.findall(r'\.([a-zA-Z_-][a-zA-Z0-9_-]*)', selector)
# Extract id (everything after #)
id_match = re.search(r'#([a-zA-Z_-][a-zA-Z0-9_-]*)', selector)
element_id = id_match.group(1) if id_match else None
# Check tag match
if tag_name and self.tag.lower() != tag_name.lower():
return False
# Check class match
if classes:
node_classes = self.get_classes()
for cls in classes:
if cls not in node_classes:
return False
# Check id match
if element_id and self.get_id() != element_id:
return False
return True
[docs]
def get_text(self, separator: str = " ", strip: bool = True) -> str:
"""Extract all text content from this node and descendants."""
texts = []
self._collect_text(texts)
result = separator.join(texts)
return result.strip() if strip else result
def _collect_text(self, texts: List[str]) -> None:
"""Recursively collect text content."""
if self.text:
text = self.text.strip()
if text:
texts.append(text)
for child in self.children:
child._collect_text(texts)
[docs]
def to_html(self) -> str:
"""Convert node back to HTML string."""
if self.tag == '_text':
return self.text
# Build opening tag
attrs_str = ""
for key, value in self.attrs.items():
if value is None:
attrs_str += f' {key}'
else:
attrs_str += f' {key}="{value}"'
# Self-closing tags
void_elements = {'area', 'base', 'br', 'col', 'embed', 'hr', 'img',
'input', 'link', 'meta', 'param', 'source', 'track', 'wbr'}
if self.tag.lower() in void_elements:
return f'<{self.tag}{attrs_str}>'
# Build content
content = self.text
for child in self.children:
content += child.to_html()
return f'<{self.tag}{attrs_str}>{content}</{self.tag}>'
[docs]
class SimpleHTMLParser(HTMLParser):
"""
Simple HTML parser that builds a basic DOM tree.
Uses Python's stdlib html.parser for parsing.
"""
def __init__(self):
super().__init__()
self.root = HTMLNode('_root')
self.current = self.root
self._stack: List[HTMLNode] = [self.root]
[docs]
def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
node = HTMLNode(tag, attrs)
node.parent = self.current
self.current.children.append(node)
# Don't push void elements onto stack
void_elements = {'area', 'base', 'br', 'col', 'embed', 'hr', 'img',
'input', 'link', 'meta', 'param', 'source', 'track', 'wbr'}
if tag.lower() not in void_elements:
self._stack.append(node)
self.current = node
[docs]
def handle_endtag(self, tag: str) -> None:
# Pop back to matching start tag
while len(self._stack) > 1:
popped = self._stack.pop()
self.current = self._stack[-1]
if popped.tag.lower() == tag.lower():
break
[docs]
def handle_data(self, data: str) -> None:
if data.strip():
# Add text to current node
self.current.text += data
[docs]
def get_root(self) -> HTMLNode:
return self.root
[docs]
def parse_html(html: str) -> HTMLNode:
"""Parse HTML string into a node tree."""
parser = SimpleHTMLParser()
parser.feed(html)
return parser.get_root()
[docs]
def select_elements(html: str, selector: str) -> List[HTMLNode]:
"""
Select elements matching a CSS selector.
Args:
html: HTML string to parse
selector: CSS selector (tag, .class, #id, or combinations)
Returns:
List of matching HTMLNode elements
"""
root = parse_html(html)
matches = []
_find_matching(root, selector, matches)
return matches
def _find_matching(node: HTMLNode, selector: str, matches: List[HTMLNode]) -> None:
"""Recursively find nodes matching selector."""
if node.tag != '_root' and node.tag != '_text':
if node.matches_selector(selector):
matches.append(node)
for child in node.children:
_find_matching(child, selector, matches)
[docs]
def get_text(html: str, separator: str = " ", strip: bool = True) -> str:
"""
Extract all text content from HTML, stripping tags.
Args:
html: HTML string
separator: String to join text segments
strip: Whether to strip whitespace
Returns:
Plain text content
"""
root = parse_html(html)
return root.get_text(separator, strip)