|
|
@@ -0,0 +1,1318 @@
|
|
|
+from __future__ import annotations
|
|
|
+from typing import Sequence, Iterable
|
|
|
+# from pandas._typing import FilePath, BaseBuffer, StorageOptions
|
|
|
+from pandas._typing import *
|
|
|
+from loguru import logger
|
|
|
+
|
|
|
+"""
|
|
|
+对pandas.io.html的修改
|
|
|
+增加属性 **kwargs
|
|
|
+colspan_single = kwargs.get('colspan_single') # 单个单元格的colspan=Literal[None, "header", "footer", "body"]
|
|
|
+rowspan_single = kwargs.get('rowspan_single') # 单个单元格的rowspan=Literal[None, "header", "footer", "body"]
|
|
|
+number_strip = kwargs.get('number_strip') # 是否去除单个cell中数字中空格 = bool
|
|
|
+"""
|
|
|
+
|
|
|
+"""
|
|
|
+:mod:`pandas.io.html` is a module containing functionality for dealing with
|
|
|
+HTML IO.
|
|
|
+
|
|
|
+"""
|
|
|
+
|
|
|
+from collections import abc
|
|
|
+import numbers
|
|
|
+import re
|
|
|
+from re import Pattern
|
|
|
+from typing import (
|
|
|
+ TYPE_CHECKING,
|
|
|
+ Literal,
|
|
|
+ cast,
|
|
|
+)
|
|
|
+import warnings
|
|
|
+
|
|
|
+from pandas._libs import lib
|
|
|
+from pandas.compat._optional import import_optional_dependency
|
|
|
+from pandas.errors import (
|
|
|
+ AbstractMethodError,
|
|
|
+ EmptyDataError,
|
|
|
+)
|
|
|
+from pandas.util._decorators import doc
|
|
|
+from pandas.util._exceptions import find_stack_level
|
|
|
+from pandas.util._validators import check_dtype_backend
|
|
|
+
|
|
|
+from pandas.core.dtypes.common import is_list_like
|
|
|
+
|
|
|
+from pandas import isna
|
|
|
+from pandas.core.indexes.base import Index
|
|
|
+from pandas.core.indexes.multi import MultiIndex
|
|
|
+from pandas.core.series import Series
|
|
|
+from pandas.core.shared_docs import _shared_docs
|
|
|
+
|
|
|
+from pandas.io.common import (
|
|
|
+ file_exists,
|
|
|
+ get_handle,
|
|
|
+ is_file_like,
|
|
|
+ is_fsspec_url,
|
|
|
+ is_url,
|
|
|
+ stringify_path,
|
|
|
+ validate_header_arg,
|
|
|
+)
|
|
|
+from pandas.io.formats.printing import pprint_thing
|
|
|
+from pandas.io.parsers import TextParser
|
|
|
+
|
|
|
+if TYPE_CHECKING:
|
|
|
+ from collections.abc import (
|
|
|
+ Iterable,
|
|
|
+ Sequence,
|
|
|
+ )
|
|
|
+
|
|
|
+ from pandas._typing import (
|
|
|
+ BaseBuffer,
|
|
|
+ DtypeBackend,
|
|
|
+ FilePath,
|
|
|
+ HTMLFlavors,
|
|
|
+ ReadBuffer,
|
|
|
+ StorageOptions,
|
|
|
+ )
|
|
|
+
|
|
|
+ from pandas import DataFrame
|
|
|
+
|
|
|
+#############
|
|
|
+# READ HTML #
|
|
|
+#############
|
|
|
+_RE_WHITESPACE = re.compile(r"[\r\n]+|\s{2,}")
|
|
|
+
|
|
|
+
|
|
|
+def _remove_whitespace(s: str, regex: Pattern = _RE_WHITESPACE) -> str:
|
|
|
+ """
|
|
|
+ Replace extra whitespace inside of a string with a single space.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ s : str or unicode
|
|
|
+ The string from which to remove extra whitespace.
|
|
|
+ regex : re.Pattern
|
|
|
+ The regular expression to use to remove extra whitespace.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ subd : str or unicode
|
|
|
+ `s` with all extra whitespace replaced with a single space.
|
|
|
+ """
|
|
|
+ return regex.sub(" ", s.strip())
|
|
|
+
|
|
|
+
|
|
|
+def _get_skiprows(skiprows: int | Sequence[int] | slice | None) -> int | Sequence[int]:
|
|
|
+ """
|
|
|
+ Get an iterator given an integer, slice or container.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ skiprows : int, slice, container
|
|
|
+ The iterator to use to skip rows; can also be a slice.
|
|
|
+
|
|
|
+ Raises
|
|
|
+ ------
|
|
|
+ TypeError
|
|
|
+ * If `skiprows` is not a slice, integer, or Container
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ it : iterable
|
|
|
+ A proper iterator to use to skip rows of a DataFrame.
|
|
|
+ """
|
|
|
+ if isinstance(skiprows, slice):
|
|
|
+ start, step = skiprows.start or 0, skiprows.step or 1
|
|
|
+ return list(range(start, skiprows.stop, step))
|
|
|
+ elif isinstance(skiprows, numbers.Integral) or is_list_like(skiprows):
|
|
|
+ return cast("int | Sequence[int]", skiprows)
|
|
|
+ elif skiprows is None:
|
|
|
+ return 0
|
|
|
+ raise TypeError(f"{type(skiprows).__name__} is not a valid type for skipping rows")
|
|
|
+
|
|
|
+
|
|
|
+def _read(
|
|
|
+ obj: FilePath | BaseBuffer,
|
|
|
+ encoding: str | None,
|
|
|
+ storage_options: StorageOptions | None,
|
|
|
+) -> str | bytes:
|
|
|
+ """
|
|
|
+ Try to read from a url, file or string.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ obj : str, unicode, path object, or file-like object
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ raw_text : str
|
|
|
+ """
|
|
|
+ text: str | bytes
|
|
|
+ if (
|
|
|
+ is_url(obj)
|
|
|
+ or hasattr(obj, "read")
|
|
|
+ or (isinstance(obj, str) and file_exists(obj))
|
|
|
+ ):
|
|
|
+ with get_handle(
|
|
|
+ obj, "r", encoding=encoding, storage_options=storage_options
|
|
|
+ ) as handles:
|
|
|
+ text = handles.handle.read()
|
|
|
+ elif isinstance(obj, (str, bytes)):
|
|
|
+ text = obj
|
|
|
+ else:
|
|
|
+ raise TypeError(f"Cannot read object of type '{type(obj).__name__}'")
|
|
|
+ return text
|
|
|
+
|
|
|
+
|
|
|
+class _HtmlFrameParser:
|
|
|
+ """
|
|
|
+ Base class for parsers that parse HTML into DataFrames.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ io : str or file-like
|
|
|
+ This can be either a string of raw HTML, a valid URL using the HTTP,
|
|
|
+ FTP, or FILE protocols or a file-like object.
|
|
|
+
|
|
|
+ match : str or regex
|
|
|
+ The text to match in the document.
|
|
|
+
|
|
|
+ attrs : dict
|
|
|
+ List of HTML <table> element attributes to match.
|
|
|
+
|
|
|
+ encoding : str
|
|
|
+ Encoding to be used by parser
|
|
|
+
|
|
|
+ displayed_only : bool
|
|
|
+ Whether or not items with "display:none" should be ignored
|
|
|
+
|
|
|
+ extract_links : {None, "all", "header", "body", "footer"}
|
|
|
+ Table elements in the specified section(s) with <a> tags will have their
|
|
|
+ href extracted.
|
|
|
+
|
|
|
+ .. versionadded:: 1.5.0
|
|
|
+
|
|
|
+ Attributes
|
|
|
+ ----------
|
|
|
+ io : str or file-like
|
|
|
+ raw HTML, URL, or file-like object
|
|
|
+
|
|
|
+ match : regex
|
|
|
+ The text to match in the raw HTML
|
|
|
+
|
|
|
+ attrs : dict-like
|
|
|
+ A dictionary of valid table attributes to use to search for table
|
|
|
+ elements.
|
|
|
+
|
|
|
+ encoding : str
|
|
|
+ Encoding to be used by parser
|
|
|
+
|
|
|
+ displayed_only : bool
|
|
|
+ Whether or not items with "display:none" should be ignored
|
|
|
+
|
|
|
+ extract_links : {None, "all", "header", "body", "footer"}
|
|
|
+ Table elements in the specified section(s) with <a> tags will have their
|
|
|
+ href extracted.
|
|
|
+
|
|
|
+ .. versionadded:: 1.5.0
|
|
|
+
|
|
|
+ Notes
|
|
|
+ -----
|
|
|
+ To subclass this class effectively you must override the following methods:
|
|
|
+ * :func:`_build_doc`
|
|
|
+ * :func:`_attr_getter`
|
|
|
+ * :func:`_href_getter`
|
|
|
+ * :func:`_text_getter`
|
|
|
+ * :func:`_parse_td`
|
|
|
+ * :func:`_parse_thead_tr`
|
|
|
+ * :func:`_parse_tbody_tr`
|
|
|
+ * :func:`_parse_tfoot_tr`
|
|
|
+ * :func:`_parse_tables`
|
|
|
+ * :func:`_equals_tag`
|
|
|
+ See each method's respective documentation for details on their
|
|
|
+ functionality.
|
|
|
+ """
|
|
|
+
|
|
|
+ def __init__(
|
|
|
+ self,
|
|
|
+ io: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
|
|
|
+ match: str | Pattern,
|
|
|
+ attrs: dict[str, str] | None,
|
|
|
+ encoding: str,
|
|
|
+ displayed_only: bool,
|
|
|
+ extract_links: Literal[None, "header", "footer", "body", "all"],
|
|
|
+ storage_options: StorageOptions = None,
|
|
|
+ ) -> None:
|
|
|
+ self.io = io
|
|
|
+ self.match = match
|
|
|
+ self.attrs = attrs
|
|
|
+ self.encoding = encoding
|
|
|
+ self.displayed_only = displayed_only
|
|
|
+ self.extract_links = extract_links
|
|
|
+ self.storage_options = storage_options
|
|
|
+
|
|
|
+ def parse_tables(self, custom_args):
|
|
|
+ """
|
|
|
+ Parse and return all tables from the DOM.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of parsed (header, body, footer) tuples from tables.
|
|
|
+ """
|
|
|
+ tables = self._parse_tables(self._build_doc(), self.match, self.attrs)
|
|
|
+ return (self._parse_thead_tbody_tfoot(table, custom_args) for table in tables)
|
|
|
+
|
|
|
+ def _attr_getter(self, obj, attr):
|
|
|
+ """
|
|
|
+ Return the attribute value of an individual DOM node.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ obj : node-like
|
|
|
+ A DOM node.
|
|
|
+
|
|
|
+ attr : str or unicode
|
|
|
+ The attribute, such as "colspan"
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ str or unicode
|
|
|
+ The attribute value.
|
|
|
+ """
|
|
|
+ # Both lxml and BeautifulSoup have the same implementation:
|
|
|
+ return obj.get(attr)
|
|
|
+
|
|
|
+ def _href_getter(self, obj) -> str | None:
|
|
|
+ """
|
|
|
+ Return a href if the DOM node contains a child <a> or None.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ obj : node-like
|
|
|
+ A DOM node.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ href : str or unicode
|
|
|
+ The href from the <a> child of the DOM node.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _text_getter(self, obj):
|
|
|
+ """
|
|
|
+ Return the text of an individual DOM node.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ obj : node-like
|
|
|
+ A DOM node.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ text : str or unicode
|
|
|
+ The text from an individual DOM node.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _parse_td(self, obj):
|
|
|
+ """
|
|
|
+ Return the td elements from a row element.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ obj : node-like
|
|
|
+ A DOM <tr> node.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of node-like
|
|
|
+ These are the elements of each row, i.e., the columns.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _parse_thead_tr(self, table):
|
|
|
+ """
|
|
|
+ Return the list of thead row elements from the parsed table element.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ table : a table element that contains zero or more thead elements.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of node-like
|
|
|
+ These are the <tr> row elements of a table.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _parse_tbody_tr(self, table):
|
|
|
+ """
|
|
|
+ Return the list of tbody row elements from the parsed table element.
|
|
|
+
|
|
|
+ HTML5 table bodies consist of either 0 or more <tbody> elements (which
|
|
|
+ only contain <tr> elements) or 0 or more <tr> elements. This method
|
|
|
+ checks for both structures.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ table : a table element that contains row elements.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of node-like
|
|
|
+ These are the <tr> row elements of a table.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _parse_tfoot_tr(self, table):
|
|
|
+ """
|
|
|
+ Return the list of tfoot row elements from the parsed table element.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ table : a table element that contains row elements.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of node-like
|
|
|
+ These are the <tr> row elements of a table.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _parse_tables(self, document, match, attrs):
|
|
|
+ """
|
|
|
+ Return all tables from the parsed DOM.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ document : the DOM from which to parse the table element.
|
|
|
+
|
|
|
+ match : str or regular expression
|
|
|
+ The text to search for in the DOM tree.
|
|
|
+
|
|
|
+ attrs : dict
|
|
|
+ A dictionary of table attributes that can be used to disambiguate
|
|
|
+ multiple tables on a page.
|
|
|
+
|
|
|
+ Raises
|
|
|
+ ------
|
|
|
+ ValueError : `match` does not match any text in the document.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of node-like
|
|
|
+ HTML <table> elements to be parsed into raw data.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _equals_tag(self, obj, tag) -> bool:
|
|
|
+ """
|
|
|
+ Return whether an individual DOM node matches a tag
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ obj : node-like
|
|
|
+ A DOM node.
|
|
|
+
|
|
|
+ tag : str
|
|
|
+ Tag name to be checked for equality.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ boolean
|
|
|
+ Whether `obj`'s tag name is `tag`
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _build_doc(self):
|
|
|
+ """
|
|
|
+ Return a tree-like object that can be used to iterate over the DOM.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ node-like
|
|
|
+ The DOM from which to parse the table element.
|
|
|
+ """
|
|
|
+ raise AbstractMethodError(self)
|
|
|
+
|
|
|
+ def _parse_thead_tbody_tfoot(self, table_html, custom_args):
|
|
|
+ """
|
|
|
+ Given a table, return parsed header, body, and foot.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ table_html : node-like
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ tuple of (header, body, footer), each a list of list-of-text rows.
|
|
|
+
|
|
|
+ Notes
|
|
|
+ -----
|
|
|
+ Header and body are lists-of-lists. Top level list is a list of
|
|
|
+ rows. Each row is a list of str text.
|
|
|
+
|
|
|
+ Logic: Use <thead>, <tbody>, <tfoot> elements to identify
|
|
|
+ header, body, and footer, otherwise:
|
|
|
+ - Put all rows into body
|
|
|
+ - Move rows from top of body to header only if
|
|
|
+ all elements inside row are <th>
|
|
|
+ - Move rows from bottom of body to footer only if
|
|
|
+ all elements inside row are <th>
|
|
|
+ """
|
|
|
+ header_rows = self._parse_thead_tr(table_html)
|
|
|
+ body_rows = self._parse_tbody_tr(table_html)
|
|
|
+ footer_rows = self._parse_tfoot_tr(table_html)
|
|
|
+
|
|
|
+ def row_is_all_th(row):
|
|
|
+ return all(self._equals_tag(t, "th") for t in self._parse_td(row))
|
|
|
+
|
|
|
+ if not header_rows:
|
|
|
+ # The table has no <thead>. Move the top all-<th> rows from
|
|
|
+ # body_rows to header_rows. (This is a common case because many
|
|
|
+ # tables in the wild have no <thead> or <tfoot>
|
|
|
+ while body_rows and row_is_all_th(body_rows[0]):
|
|
|
+ header_rows.append(body_rows.pop(0))
|
|
|
+
|
|
|
+ header = self._expand_colspan_rowspan(header_rows, section="header", custom_args=custom_args)
|
|
|
+ body = self._expand_colspan_rowspan(body_rows, section="body", custom_args=custom_args)
|
|
|
+ footer = self._expand_colspan_rowspan(footer_rows, section="footer", custom_args=custom_args)
|
|
|
+
|
|
|
+ return header, body, footer
|
|
|
+
|
|
|
+ def _expand_colspan_rowspan(
|
|
|
+ self, rows, section: Literal["header", "footer", "body"], custom_args
|
|
|
+ ):
|
|
|
+ """
|
|
|
+ Given a list of <tr>s, return a list of text rows.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ rows : list of node-like
|
|
|
+ List of <tr>s
|
|
|
+ section : the section that the rows belong to (header, body or footer).
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of list
|
|
|
+ Each returned row is a list of str text, or tuple (text, link)
|
|
|
+ if extract_links is not None.
|
|
|
+
|
|
|
+ Notes
|
|
|
+ -----
|
|
|
+ Any cell with ``rowspan`` or ``colspan`` will have its contents copied
|
|
|
+ to subsequent cells.
|
|
|
+ """
|
|
|
+ colspan_single = custom_args.get('colspan_single')
|
|
|
+ rowspan_single = custom_args.get('rowspan_single')
|
|
|
+ number_strip = custom_args.get('number_strip')
|
|
|
+
|
|
|
+ all_texts = [] # list of rows, each a list of str
|
|
|
+ text: str | tuple
|
|
|
+ remainder: list[
|
|
|
+ tuple[int, str | tuple, int]
|
|
|
+ ] = [] # list of (index, text, nrows)
|
|
|
+
|
|
|
+ for tr in rows:
|
|
|
+ texts = [] # the output for this row
|
|
|
+ next_remainder = []
|
|
|
+
|
|
|
+ index = 0
|
|
|
+ tds = self._parse_td(tr)
|
|
|
+ for td in tds:
|
|
|
+ # Append texts from previous rows with rowspan>1 that come
|
|
|
+ # before this <td>
|
|
|
+ while remainder and remainder[0][0] <= index:
|
|
|
+ prev_i, prev_text, prev_rowspan = remainder.pop(0)
|
|
|
+ texts.append(prev_text)
|
|
|
+ if prev_rowspan > 1:
|
|
|
+ next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
|
|
|
+ index += 1
|
|
|
+
|
|
|
+ # Append the text from this <td>, colspan times
|
|
|
+ text = _remove_whitespace(self._text_getter(td))
|
|
|
+ if self.extract_links in ("all", section):
|
|
|
+ href = self._href_getter(td)
|
|
|
+ text = (text, href)
|
|
|
+ rowspan = int(self._attr_getter(td, "rowspan") or 1)
|
|
|
+ colspan = int(self._attr_getter(td, "colspan") or 1)
|
|
|
+
|
|
|
+ for i in range(colspan):
|
|
|
+ if i == 0:
|
|
|
+ texts.append(text)
|
|
|
+ elif section in colspan_single:
|
|
|
+ texts.append('')
|
|
|
+ else:
|
|
|
+ texts.append(text)
|
|
|
+
|
|
|
+ if rowspan > 1 and section not in rowspan_single:
|
|
|
+ next_remainder.append((index, text, rowspan - 1))
|
|
|
+ index += 1
|
|
|
+
|
|
|
+ # Append texts from previous rows at the final position
|
|
|
+ for prev_i, prev_text, prev_rowspan in remainder:
|
|
|
+ texts.append(prev_text)
|
|
|
+ if prev_rowspan > 1:
|
|
|
+ next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
|
|
|
+
|
|
|
+ all_texts.append(texts)
|
|
|
+ remainder = next_remainder
|
|
|
+
|
|
|
+ # Append rows that only appear because the previous row had non-1
|
|
|
+ # rowspan
|
|
|
+ while remainder:
|
|
|
+ next_remainder = []
|
|
|
+ texts = []
|
|
|
+ for prev_i, prev_text, prev_rowspan in remainder:
|
|
|
+ texts.append(prev_text)
|
|
|
+ if prev_rowspan > 1:
|
|
|
+ next_remainder.append((prev_i, prev_text, prev_rowspan - 1))
|
|
|
+ all_texts.append(texts)
|
|
|
+ remainder = next_remainder
|
|
|
+
|
|
|
+ return all_texts
|
|
|
+
|
|
|
+ def _handle_hidden_tables(self, tbl_list, attr_name: str):
|
|
|
+ """
|
|
|
+ Return list of tables, potentially removing hidden elements
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ tbl_list : list of node-like
|
|
|
+ Type of list elements will vary depending upon parser used
|
|
|
+ attr_name : str
|
|
|
+ Name of the accessor for retrieving HTML attributes
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ list of node-like
|
|
|
+ Return type matches `tbl_list`
|
|
|
+ """
|
|
|
+ if not self.displayed_only:
|
|
|
+ return tbl_list
|
|
|
+
|
|
|
+ return [
|
|
|
+ x
|
|
|
+ for x in tbl_list
|
|
|
+ if "display:none"
|
|
|
+ not in getattr(x, attr_name).get("style", "").replace(" ", "")
|
|
|
+ ]
|
|
|
+
|
|
|
+
|
|
|
+class _BeautifulSoupHtml5LibFrameParser(_HtmlFrameParser):
|
|
|
+ """
|
|
|
+ HTML to DataFrame parser that uses BeautifulSoup under the hood.
|
|
|
+
|
|
|
+ See Also
|
|
|
+ --------
|
|
|
+ pandas.io.html._HtmlFrameParser
|
|
|
+ pandas.io.html._LxmlFrameParser
|
|
|
+
|
|
|
+ Notes
|
|
|
+ -----
|
|
|
+ Documentation strings for this class are in the base class
|
|
|
+ :class:`pandas.io.html._HtmlFrameParser`.
|
|
|
+ """
|
|
|
+
|
|
|
+ def _parse_tables(self, document, match, attrs):
|
|
|
+ element_name = "table"
|
|
|
+ tables = document.find_all(element_name, attrs=attrs)
|
|
|
+ if not tables:
|
|
|
+ raise ValueError("No tables found")
|
|
|
+
|
|
|
+ result = []
|
|
|
+ unique_tables = set()
|
|
|
+ tables = self._handle_hidden_tables(tables, "attrs")
|
|
|
+
|
|
|
+ for table in tables:
|
|
|
+ if self.displayed_only:
|
|
|
+ for elem in table.find_all("style"):
|
|
|
+ elem.decompose()
|
|
|
+
|
|
|
+ for elem in table.find_all(style=re.compile(r"display:\s*none")):
|
|
|
+ elem.decompose()
|
|
|
+
|
|
|
+ if table not in unique_tables and table.find(string=match) is not None:
|
|
|
+ result.append(table)
|
|
|
+ unique_tables.add(table)
|
|
|
+ if not result:
|
|
|
+ raise ValueError(f"No tables found matching pattern {repr(match.pattern)}")
|
|
|
+ return result
|
|
|
+
|
|
|
+ def _href_getter(self, obj) -> str | None:
|
|
|
+ a = obj.find("a", href=True)
|
|
|
+ return None if not a else a["href"]
|
|
|
+
|
|
|
+ def _text_getter(self, obj):
|
|
|
+ return obj.text
|
|
|
+
|
|
|
+ def _equals_tag(self, obj, tag) -> bool:
|
|
|
+ return obj.name == tag
|
|
|
+
|
|
|
+ def _parse_td(self, row):
|
|
|
+ return row.find_all(("td", "th"), recursive=False)
|
|
|
+
|
|
|
+ def _parse_thead_tr(self, table):
|
|
|
+ return table.select("thead tr")
|
|
|
+
|
|
|
+ def _parse_tbody_tr(self, table):
|
|
|
+ from_tbody = table.select("tbody tr")
|
|
|
+ from_root = table.find_all("tr", recursive=False)
|
|
|
+ # HTML spec: at most one of these lists has content
|
|
|
+ return from_tbody + from_root
|
|
|
+
|
|
|
+ def _parse_tfoot_tr(self, table):
|
|
|
+ return table.select("tfoot tr")
|
|
|
+
|
|
|
+ def _setup_build_doc(self):
|
|
|
+ raw_text = _read(self.io, self.encoding, self.storage_options)
|
|
|
+ if not raw_text:
|
|
|
+ raise ValueError(f"No text parsed from document: {self.io}")
|
|
|
+ return raw_text
|
|
|
+
|
|
|
+ def _build_doc(self):
|
|
|
+ from bs4 import BeautifulSoup
|
|
|
+
|
|
|
+ bdoc = self._setup_build_doc()
|
|
|
+ if isinstance(bdoc, bytes) and self.encoding is not None:
|
|
|
+ udoc = bdoc.decode(self.encoding)
|
|
|
+ from_encoding = None
|
|
|
+ else:
|
|
|
+ udoc = bdoc
|
|
|
+ from_encoding = self.encoding
|
|
|
+
|
|
|
+ soup = BeautifulSoup(udoc, features="html5lib", from_encoding=from_encoding)
|
|
|
+
|
|
|
+ for br in soup.find_all("br"):
|
|
|
+ br.replace_with("\n" + br.text)
|
|
|
+
|
|
|
+ return soup
|
|
|
+
|
|
|
+
|
|
|
+def _build_xpath_expr(attrs) -> str:
|
|
|
+ """
|
|
|
+ Build an xpath expression to simulate bs4's ability to pass in kwargs to
|
|
|
+ search for attributes when using the lxml parser.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ attrs : dict
|
|
|
+ A dict of HTML attributes. These are NOT checked for validity.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ expr : unicode
|
|
|
+ An XPath expression that checks for the given HTML attributes.
|
|
|
+ """
|
|
|
+ # give class attribute as class_ because class is a python keyword
|
|
|
+ if "class_" in attrs:
|
|
|
+ attrs["class"] = attrs.pop("class_")
|
|
|
+
|
|
|
+ s = " and ".join([f"@{k}={repr(v)}" for k, v in attrs.items()])
|
|
|
+ return f"[{s}]"
|
|
|
+
|
|
|
+
|
|
|
+_re_namespace = {"re": "http://exslt.org/regular-expressions"}
|
|
|
+
|
|
|
+
|
|
|
+class _LxmlFrameParser(_HtmlFrameParser):
|
|
|
+ """
|
|
|
+ HTML to DataFrame parser that uses lxml under the hood.
|
|
|
+
|
|
|
+ Warning
|
|
|
+ -------
|
|
|
+ This parser can only handle HTTP, FTP, and FILE urls.
|
|
|
+
|
|
|
+ See Also
|
|
|
+ --------
|
|
|
+ _HtmlFrameParser
|
|
|
+ _BeautifulSoupLxmlFrameParser
|
|
|
+
|
|
|
+ Notes
|
|
|
+ -----
|
|
|
+ Documentation strings for this class are in the base class
|
|
|
+ :class:`_HtmlFrameParser`.
|
|
|
+ """
|
|
|
+
|
|
|
+ def _href_getter(self, obj) -> str | None:
|
|
|
+ href = obj.xpath(".//a/@href")
|
|
|
+ return None if not href else href[0]
|
|
|
+
|
|
|
+ def _text_getter(self, obj):
|
|
|
+ return obj.text_content()
|
|
|
+
|
|
|
+ def _parse_td(self, row):
|
|
|
+ # Look for direct children only: the "row" element here may be a
|
|
|
+ # <thead> or <tfoot> (see _parse_thead_tr).
|
|
|
+ return row.xpath("./td|./th")
|
|
|
+
|
|
|
+ def _parse_tables(self, document, match, kwargs):
|
|
|
+ pattern = match.pattern
|
|
|
+
|
|
|
+ # 1. check all descendants for the given pattern and only search tables
|
|
|
+ # GH 49929
|
|
|
+ xpath_expr = f"//table[.//text()[re:test(., {repr(pattern)})]]"
|
|
|
+
|
|
|
+ # if any table attributes were given build an xpath expression to
|
|
|
+ # search for them
|
|
|
+ if kwargs:
|
|
|
+ xpath_expr += _build_xpath_expr(kwargs)
|
|
|
+
|
|
|
+ tables = document.xpath(xpath_expr, namespaces=_re_namespace)
|
|
|
+
|
|
|
+ tables = self._handle_hidden_tables(tables, "attrib")
|
|
|
+ if self.displayed_only:
|
|
|
+ for table in tables:
|
|
|
+ # lxml utilizes XPATH 1.0 which does not have regex
|
|
|
+ # support. As a result, we find all elements with a style
|
|
|
+ # attribute and iterate them to check for display:none
|
|
|
+ for elem in table.xpath(".//style"):
|
|
|
+ elem.drop_tree()
|
|
|
+ for elem in table.xpath(".//*[@style]"):
|
|
|
+ if "display:none" in elem.attrib.get("style", "").replace(" ", ""):
|
|
|
+ elem.drop_tree()
|
|
|
+ if not tables:
|
|
|
+ raise ValueError(f"No tables found matching regex {repr(pattern)}")
|
|
|
+ return tables
|
|
|
+
|
|
|
+ def _equals_tag(self, obj, tag) -> bool:
|
|
|
+ return obj.tag == tag
|
|
|
+
|
|
|
+ def _build_doc(self):
|
|
|
+ """
|
|
|
+ Raises
|
|
|
+ ------
|
|
|
+ ValueError
|
|
|
+ * If a URL that lxml cannot parse is passed.
|
|
|
+
|
|
|
+ Exception
|
|
|
+ * Any other ``Exception`` thrown. For example, trying to parse a
|
|
|
+ URL that is syntactically correct on a machine with no internet
|
|
|
+ connection will fail.
|
|
|
+
|
|
|
+ See Also
|
|
|
+ --------
|
|
|
+ pandas.io.html._HtmlFrameParser._build_doc
|
|
|
+ """
|
|
|
+ from lxml.etree import XMLSyntaxError
|
|
|
+ from lxml.html import (
|
|
|
+ HTMLParser,
|
|
|
+ fromstring,
|
|
|
+ parse,
|
|
|
+ )
|
|
|
+
|
|
|
+ parser = HTMLParser(recover=True, encoding=self.encoding)
|
|
|
+
|
|
|
+ try:
|
|
|
+ if is_url(self.io):
|
|
|
+ with get_handle(
|
|
|
+ self.io, "r", storage_options=self.storage_options
|
|
|
+ ) as f:
|
|
|
+ r = parse(f.handle, parser=parser)
|
|
|
+ else:
|
|
|
+ # try to parse the input in the simplest way
|
|
|
+ r = parse(self.io, parser=parser)
|
|
|
+ try:
|
|
|
+ r = r.getroot()
|
|
|
+ except AttributeError:
|
|
|
+ pass
|
|
|
+ except (UnicodeDecodeError, OSError) as e:
|
|
|
+ # if the input is a blob of html goop
|
|
|
+ if not is_url(self.io):
|
|
|
+ r = fromstring(self.io, parser=parser)
|
|
|
+
|
|
|
+ try:
|
|
|
+ r = r.getroot()
|
|
|
+ except AttributeError:
|
|
|
+ pass
|
|
|
+ else:
|
|
|
+ raise e
|
|
|
+ else:
|
|
|
+ if not hasattr(r, "text_content"):
|
|
|
+ raise XMLSyntaxError("no text parsed from document", 0, 0, 0)
|
|
|
+
|
|
|
+ for br in r.xpath("*//br"):
|
|
|
+ br.tail = "\n" + (br.tail or "")
|
|
|
+
|
|
|
+ return r
|
|
|
+
|
|
|
+ def _parse_thead_tr(self, table):
|
|
|
+ rows = []
|
|
|
+
|
|
|
+ for thead in table.xpath(".//thead"):
|
|
|
+ rows.extend(thead.xpath("./tr"))
|
|
|
+
|
|
|
+ # HACK: lxml does not clean up the clearly-erroneous
|
|
|
+ # <thead><th>foo</th><th>bar</th></thead>. (Missing <tr>). Add
|
|
|
+ # the <thead> and _pretend_ it's a <tr>; _parse_td() will find its
|
|
|
+ # children as though it's a <tr>.
|
|
|
+ #
|
|
|
+ # Better solution would be to use html5lib.
|
|
|
+ elements_at_root = thead.xpath("./td|./th")
|
|
|
+ if elements_at_root:
|
|
|
+ rows.append(thead)
|
|
|
+
|
|
|
+ return rows
|
|
|
+
|
|
|
+ def _parse_tbody_tr(self, table):
|
|
|
+ from_tbody = table.xpath(".//tbody//tr")
|
|
|
+ from_root = table.xpath("./tr")
|
|
|
+ # HTML spec: at most one of these lists has content
|
|
|
+ return from_tbody + from_root
|
|
|
+
|
|
|
+ def _parse_tfoot_tr(self, table):
|
|
|
+ return table.xpath(".//tfoot//tr")
|
|
|
+
|
|
|
+
|
|
|
+def _expand_elements(body) -> None:
|
|
|
+ data = [len(elem) for elem in body]
|
|
|
+ lens = Series(data)
|
|
|
+ lens_max = lens.max()
|
|
|
+ not_max = lens[lens != lens_max]
|
|
|
+
|
|
|
+ empty = [""]
|
|
|
+ for ind, length in not_max.items():
|
|
|
+ body[ind] += empty * (lens_max - length)
|
|
|
+
|
|
|
+
|
|
|
+def _data_to_frame(**kwargs):
|
|
|
+
|
|
|
+ def clean_number_string(cell_value):
|
|
|
+ # 正则表达式，用于匹配数字之间的空格
|
|
|
+ # PATTERN = re.compile(r'(\d)\s+(\d)')
|
|
|
+ # 检查是否只包含允许的字符
|
|
|
+ allowed_chars = set("0123456789.,\n\t+- ")
|
|
|
+ if cell_value is None or cell_value == '':
|
|
|
+ return cell_value
|
|
|
+ if any(char not in allowed_chars for char in cell_value):
|
|
|
+ return cell_value
|
|
|
+
|
|
|
+ # 去除所有空白字符
|
|
|
+ cleaned = ''.join(char for char in cell_value if char not in " \t\n")
|
|
|
+ # 将逗号移除
|
|
|
+ cleaned = cleaned.replace(',', '')
|
|
|
+ return cleaned
|
|
|
+
|
|
|
+ def convert_to_float(cleaned_value):
|
|
|
+ try:
|
|
|
+ # 尝试将清理后的字符串转换为浮点数
|
|
|
+ number = float(cleaned_value)
|
|
|
+ return number
|
|
|
+ except ValueError:
|
|
|
+ # 如果转换失败，返回原始值或适当的错误信息
|
|
|
+ logger.error(f"无法将 '{cleaned_value}' 转换为浮点数")
|
|
|
+ # 返回空值
|
|
|
+ return None
|
|
|
+
|
|
|
+ head, body, foot = kwargs.pop("data")
|
|
|
+ header = kwargs.pop("header")
|
|
|
+ kwargs["skiprows"] = _get_skiprows(kwargs["skiprows"])
|
|
|
+ number_strip = kwargs.get('number_strip')
|
|
|
+ if head:
|
|
|
+ body = head + body
|
|
|
+
|
|
|
+ # Infer header when there is a <thead> or top <th>-only rows
|
|
|
+ if header is None:
|
|
|
+ if len(head) == 1:
|
|
|
+ header = 0
|
|
|
+ else:
|
|
|
+ # ignore all-empty-text rows
|
|
|
+ header = [i for i, row in enumerate(head) if any(text for text in row)]
|
|
|
+
|
|
|
+ if foot:
|
|
|
+ body += foot
|
|
|
+
|
|
|
+ # fill out elements of body that are "ragged"
|
|
|
+ _expand_elements(body)
|
|
|
+ # 如果 number_strip 为 True，则去除文本中的数字前后以及中间的空格
|
|
|
+ if number_strip:
|
|
|
+ for i in range(len(body)):
|
|
|
+ for j in range(len(body[i])):
|
|
|
+ body[i][j] = clean_number_string(body[i][j])
|
|
|
+ with TextParser(body, header=header, **kwargs) as tp:
|
|
|
+ return tp.read()
|
|
|
+
|
|
|
+_valid_parsers = {
|
|
|
+ "lxml": _LxmlFrameParser,
|
|
|
+ None: _LxmlFrameParser,
|
|
|
+ "html5lib": _BeautifulSoupHtml5LibFrameParser,
|
|
|
+ "bs4": _BeautifulSoupHtml5LibFrameParser,
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
+def _parser_dispatch(flavor: HTMLFlavors | None) -> type[_HtmlFrameParser]:
|
|
|
+ """
|
|
|
+ Choose the parser based on the input flavor.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ flavor : {{"lxml", "html5lib", "bs4"}} or None
|
|
|
+ The type of parser to use. This must be a valid backend.
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ cls : _HtmlFrameParser subclass
|
|
|
+ The parser class based on the requested input flavor.
|
|
|
+
|
|
|
+ Raises
|
|
|
+ ------
|
|
|
+ ValueError
|
|
|
+ * If `flavor` is not a valid backend.
|
|
|
+ ImportError
|
|
|
+ * If you do not have the requested `flavor`
|
|
|
+ """
|
|
|
+ valid_parsers = list(_valid_parsers.keys())
|
|
|
+ if flavor not in valid_parsers:
|
|
|
+ raise ValueError(
|
|
|
+ f"{repr(flavor)} is not a valid flavor, valid flavors are {valid_parsers}"
|
|
|
+ )
|
|
|
+
|
|
|
+ if flavor in ("bs4", "html5lib"):
|
|
|
+ import_optional_dependency("html5lib")
|
|
|
+ import_optional_dependency("bs4")
|
|
|
+ else:
|
|
|
+ import_optional_dependency("lxml.etree")
|
|
|
+ return _valid_parsers[flavor]
|
|
|
+
|
|
|
+
|
|
|
+def _print_as_set(s) -> str:
|
|
|
+ arg = ", ".join([pprint_thing(el) for el in s])
|
|
|
+ return f"{{{arg}}}"
|
|
|
+
|
|
|
+
|
|
|
+def _validate_flavor(flavor):
|
|
|
+ if flavor is None:
|
|
|
+ flavor = "lxml", "bs4"
|
|
|
+ elif isinstance(flavor, str):
|
|
|
+ flavor = (flavor,)
|
|
|
+ elif isinstance(flavor, abc.Iterable):
|
|
|
+ if not all(isinstance(flav, str) for flav in flavor):
|
|
|
+ raise TypeError(
|
|
|
+ f"Object of type {repr(type(flavor).__name__)} "
|
|
|
+ f"is not an iterable of strings"
|
|
|
+ )
|
|
|
+ else:
|
|
|
+ msg = repr(flavor) if isinstance(flavor, str) else str(flavor)
|
|
|
+ msg += " is not a valid flavor"
|
|
|
+ raise ValueError(msg)
|
|
|
+
|
|
|
+ flavor = tuple(flavor)
|
|
|
+ valid_flavors = set(_valid_parsers)
|
|
|
+ flavor_set = set(flavor)
|
|
|
+
|
|
|
+ if not flavor_set & valid_flavors:
|
|
|
+ raise ValueError(
|
|
|
+ f"{_print_as_set(flavor_set)} is not a valid set of flavors, valid "
|
|
|
+ f"flavors are {_print_as_set(valid_flavors)}"
|
|
|
+ )
|
|
|
+ return flavor
|
|
|
+
|
|
|
+
|
|
|
+def _parse(
|
|
|
+ flavor,
|
|
|
+ io,
|
|
|
+ match,
|
|
|
+ attrs,
|
|
|
+ encoding,
|
|
|
+ displayed_only,
|
|
|
+ extract_links,
|
|
|
+ storage_options,
|
|
|
+ custom_args,
|
|
|
+ **kwargs,
|
|
|
+):
|
|
|
+ flavor = _validate_flavor(flavor)
|
|
|
+ compiled_match = re.compile(match) # you can pass a compiled regex here
|
|
|
+
|
|
|
+ retained = None
|
|
|
+ for flav in flavor:
|
|
|
+ parser = _parser_dispatch(flav)
|
|
|
+ p = parser(
|
|
|
+ io,
|
|
|
+ compiled_match,
|
|
|
+ attrs,
|
|
|
+ encoding,
|
|
|
+ displayed_only,
|
|
|
+ extract_links,
|
|
|
+ storage_options,
|
|
|
+ )
|
|
|
+
|
|
|
+ try:
|
|
|
+ tables = p.parse_tables(custom_args)
|
|
|
+ except ValueError as caught:
|
|
|
+ # if `io` is an io-like object, check if it's seekable
|
|
|
+ # and try to rewind it before trying the next parser
|
|
|
+ if hasattr(io, "seekable") and io.seekable():
|
|
|
+ io.seek(0)
|
|
|
+ elif hasattr(io, "seekable") and not io.seekable():
|
|
|
+ # if we couldn't rewind it, let the user know
|
|
|
+ raise ValueError(
|
|
|
+ f"The flavor {flav} failed to parse your input. "
|
|
|
+ "Since you passed a non-rewindable file "
|
|
|
+ "object, we can't rewind it to try "
|
|
|
+ "another parser. Try read_html() with a different flavor."
|
|
|
+ ) from caught
|
|
|
+
|
|
|
+ retained = caught
|
|
|
+ else:
|
|
|
+ break
|
|
|
+ else:
|
|
|
+ assert retained is not None # for mypy
|
|
|
+ raise retained
|
|
|
+
|
|
|
+ ret = []
|
|
|
+ for table in tables:
|
|
|
+ try:
|
|
|
+ df = _data_to_frame(data=table, colspan_single=custom_args.get('colspan_single'), rowspan_single=custom_args.get('rowspan_single'), number_strip=custom_args.get('number_strip'), **kwargs)
|
|
|
+ # Cast MultiIndex header to an Index of tuples when extracting header
|
|
|
+ # links and replace nan with None (therefore can't use mi.to_flat_index()).
|
|
|
+ # This maintains consistency of selection (e.g. df.columns.str[1])
|
|
|
+ if extract_links in ("all", "header") and isinstance(
|
|
|
+ df.columns, MultiIndex
|
|
|
+ ):
|
|
|
+ df.columns = Index(
|
|
|
+ ((col[0], None if isna(col[1]) else col[1]) for col in df.columns),
|
|
|
+ tupleize_cols=False,
|
|
|
+ )
|
|
|
+
|
|
|
+ ret.append(df)
|
|
|
+ except EmptyDataError: # empty table
|
|
|
+ continue
|
|
|
+ return ret
|
|
|
+
|
|
|
+
|
|
|
+@doc(storage_options=_shared_docs["storage_options"])
|
|
|
+def read_html_zhch(
|
|
|
+ io: FilePath | ReadBuffer[str],
|
|
|
+ # 解释*含义：这里的*表示参数列表的开始，意味着后续的所有参数都必须是关键字参数（即必须通过参数名来传递）。
|
|
|
+ *,
|
|
|
+ match: str | Pattern = ".+",
|
|
|
+ flavor: HTMLFlavors | Sequence[HTMLFlavors] | None = None,
|
|
|
+ header: int | Sequence[int] | None = None,
|
|
|
+ index_col: int | Sequence[int] | None = None,
|
|
|
+ skiprows: int | Sequence[int] | slice | None = None,
|
|
|
+ attrs: dict[str, str] | None = None,
|
|
|
+ parse_dates: bool = False,
|
|
|
+ thousands: str | None = ",",
|
|
|
+ encoding: str | None = None,
|
|
|
+ decimal: str = ".",
|
|
|
+ converters: dict | None = None,
|
|
|
+ na_values: Iterable[object] | None = None,
|
|
|
+ keep_default_na: bool = True,
|
|
|
+ displayed_only: bool = True,
|
|
|
+ extract_links: Literal[None, "header", "footer", "body", "all"] = None,
|
|
|
+ dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
|
|
|
+ storage_options: StorageOptions = None,
|
|
|
+ custom_args: dict = {} ,
|
|
|
+) -> list[DataFrame]:
|
|
|
+ r"""
|
|
|
+ Read HTML tables into a ``list`` of ``DataFrame`` objects.
|
|
|
+
|
|
|
+ Parameters
|
|
|
+ ----------
|
|
|
+ io : str, path object, or file-like object
|
|
|
+ String, path object (implementing ``os.PathLike[str]``), or file-like
|
|
|
+ object implementing a string ``read()`` function.
|
|
|
+ The string can represent a URL or the HTML itself. Note that
|
|
|
+ lxml only accepts the http, ftp and file url protocols. If you have a
|
|
|
+ URL that starts with ``'https'`` you might try removing the ``'s'``.
|
|
|
+
|
|
|
+ .. deprecated:: 2.1.0
|
|
|
+ Passing html literal strings is deprecated.
|
|
|
+ Wrap literal string/bytes input in ``io.StringIO``/``io.BytesIO`` instead.
|
|
|
+
|
|
|
+ match : str or compiled regular expression, optional
|
|
|
+ The set of tables containing text matching this regex or string will be
|
|
|
+ returned. Unless the HTML is extremely simple you will probably need to
|
|
|
+ pass a non-empty string here. Defaults to '.+' (match any non-empty
|
|
|
+ string). The default value will return all tables contained on a page.
|
|
|
+ This value is converted to a regular expression so that there is
|
|
|
+ consistent behavior between Beautiful Soup and lxml.
|
|
|
+
|
|
|
+ flavor : {{"lxml", "html5lib", "bs4"}} or list-like, optional
|
|
|
+ The parsing engine (or list of parsing engines) to use. 'bs4' and
|
|
|
+ 'html5lib' are synonymous with each other, they are both there for
|
|
|
+ backwards compatibility. The default of ``None`` tries to use ``lxml``
|
|
|
+ to parse and if that fails it falls back on ``bs4`` + ``html5lib``.
|
|
|
+
|
|
|
+ header : int or list-like, optional
|
|
|
+ The row (or list of rows for a :class:`~pandas.MultiIndex`) to use to
|
|
|
+ make the columns headers.
|
|
|
+
|
|
|
+ index_col : int or list-like, optional
|
|
|
+ The column (or list of columns) to use to create the index.
|
|
|
+
|
|
|
+ skiprows : int, list-like or slice, optional
|
|
|
+ Number of rows to skip after parsing the column integer. 0-based. If a
|
|
|
+ sequence of integers or a slice is given, will skip the rows indexed by
|
|
|
+ that sequence. Note that a single element sequence means 'skip the nth
|
|
|
+ row' whereas an integer means 'skip n rows'.
|
|
|
+
|
|
|
+ attrs : dict, optional
|
|
|
+ This is a dictionary of attributes that you can pass to use to identify
|
|
|
+ the table in the HTML. These are not checked for validity before being
|
|
|
+ passed to lxml or Beautiful Soup. However, these attributes must be
|
|
|
+ valid HTML table attributes to work correctly. For example, ::
|
|
|
+
|
|
|
+ attrs = {{'id': 'table'}}
|
|
|
+
|
|
|
+ is a valid attribute dictionary because the 'id' HTML tag attribute is
|
|
|
+ a valid HTML attribute for *any* HTML tag as per `this document
|
|
|
+ <https://html.spec.whatwg.org/multipage/dom.html#global-attributes>`__. ::
|
|
|
+
|
|
|
+ attrs = {{'asdf': 'table'}}
|
|
|
+
|
|
|
+ is *not* a valid attribute dictionary because 'asdf' is not a valid
|
|
|
+ HTML attribute even if it is a valid XML attribute. Valid HTML 4.01
|
|
|
+ table attributes can be found `here
|
|
|
+ <http://www.w3.org/TR/REC-html40/struct/tables.html#h-11.2>`__. A
|
|
|
+ working draft of the HTML 5 spec can be found `here
|
|
|
+ <https://html.spec.whatwg.org/multipage/tables.html>`__. It contains the
|
|
|
+ latest information on table attributes for the modern web.
|
|
|
+
|
|
|
+ parse_dates : bool, optional
|
|
|
+ See :func:`~read_csv` for more details.
|
|
|
+
|
|
|
+ thousands : str, optional
|
|
|
+ Separator to use to parse thousands. Defaults to ``','``.
|
|
|
+
|
|
|
+ encoding : str, optional
|
|
|
+ The encoding used to decode the web page. Defaults to ``None``.``None``
|
|
|
+ preserves the previous encoding behavior, which depends on the
|
|
|
+ underlying parser library (e.g., the parser library will try to use
|
|
|
+ the encoding provided by the document).
|
|
|
+
|
|
|
+ decimal : str, default '.'
|
|
|
+ Character to recognize as decimal point (e.g. use ',' for European
|
|
|
+ data).
|
|
|
+
|
|
|
+ converters : dict, default None
|
|
|
+ Dict of functions for converting values in certain columns. Keys can
|
|
|
+ either be integers or column labels, values are functions that take one
|
|
|
+ input argument, the cell (not column) content, and return the
|
|
|
+ transformed content.
|
|
|
+
|
|
|
+ na_values : iterable, default None
|
|
|
+ Custom NA values.
|
|
|
+
|
|
|
+ keep_default_na : bool, default True
|
|
|
+ If na_values are specified and keep_default_na is False the default NaN
|
|
|
+ values are overridden, otherwise they're appended to.
|
|
|
+
|
|
|
+ displayed_only : bool, default True
|
|
|
+ Whether elements with "display: none" should be parsed.
|
|
|
+
|
|
|
+ extract_links : {{None, "all", "header", "body", "footer"}}
|
|
|
+ Table elements in the specified section(s) with <a> tags will have their
|
|
|
+ href extracted.
|
|
|
+
|
|
|
+ .. versionadded:: 1.5.0
|
|
|
+
|
|
|
+ dtype_backend : {{'numpy_nullable', 'pyarrow'}}, default 'numpy_nullable'
|
|
|
+ Back-end data type applied to the resultant :class:`DataFrame`
|
|
|
+ (still experimental). Behaviour is as follows:
|
|
|
+
|
|
|
+ * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
|
|
|
+ (default).
|
|
|
+ * ``"pyarrow"``: returns pyarrow-backed nullable :class:`ArrowDtype`
|
|
|
+ DataFrame.
|
|
|
+
|
|
|
+ .. versionadded:: 2.0
|
|
|
+
|
|
|
+ {storage_options}
|
|
|
+
|
|
|
+ .. versionadded:: 2.1.0
|
|
|
+
|
|
|
+ Returns
|
|
|
+ -------
|
|
|
+ dfs
|
|
|
+ A list of DataFrames.
|
|
|
+
|
|
|
+ See Also
|
|
|
+ --------
|
|
|
+ read_csv : Read a comma-separated values (csv) file into DataFrame.
|
|
|
+
|
|
|
+ Notes
|
|
|
+ -----
|
|
|
+ Before using this function you should read the :ref:`gotchas about the
|
|
|
+ HTML parsing libraries <io.html.gotchas>`.
|
|
|
+
|
|
|
+ Expect to do some cleanup after you call this function. For example, you
|
|
|
+ might need to manually assign column names if the column names are
|
|
|
+ converted to NaN when you pass the `header=0` argument. We try to assume as
|
|
|
+ little as possible about the structure of the table and push the
|
|
|
+ idiosyncrasies of the HTML contained in the table to the user.
|
|
|
+
|
|
|
+ This function searches for ``<table>`` elements and only for ``<tr>``
|
|
|
+ and ``<th>`` rows and ``<td>`` elements within each ``<tr>`` or ``<th>``
|
|
|
+ element in the table. ``<td>`` stands for "table data". This function
|
|
|
+ attempts to properly handle ``colspan`` and ``rowspan`` attributes.
|
|
|
+ If the function has a ``<thead>`` argument, it is used to construct
|
|
|
+ the header, otherwise the function attempts to find the header within
|
|
|
+ the body (by putting rows with only ``<th>`` elements into the header).
|
|
|
+
|
|
|
+ Similar to :func:`~read_csv` the `header` argument is applied
|
|
|
+ **after** `skiprows` is applied.
|
|
|
+
|
|
|
+ This function will *always* return a list of :class:`DataFrame` *or*
|
|
|
+ it will fail, e.g., it will *not* return an empty list.
|
|
|
+
|
|
|
+ Examples
|
|
|
+ --------
|
|
|
+ See the :ref:`read_html documentation in the IO section of the docs
|
|
|
+ <io.read_html>` for some examples of reading in HTML tables.
|
|
|
+ """
|
|
|
+ # Type check here. We don't want to parse only to fail because of an
|
|
|
+ # invalid value of an integer skiprows.
|
|
|
+ if isinstance(skiprows, numbers.Integral) and skiprows < 0:
|
|
|
+ raise ValueError(
|
|
|
+ "cannot skip rows starting from the end of the "
|
|
|
+ "data (you passed a negative value)"
|
|
|
+ )
|
|
|
+ if extract_links not in [None, "header", "footer", "body", "all"]:
|
|
|
+ raise ValueError(
|
|
|
+ "`extract_links` must be one of "
|
|
|
+ '{None, "header", "footer", "body", "all"}, got '
|
|
|
+ f'"{extract_links}"'
|
|
|
+ )
|
|
|
+
|
|
|
+ validate_header_arg(header)
|
|
|
+ check_dtype_backend(dtype_backend)
|
|
|
+
|
|
|
+ io = stringify_path(io)
|
|
|
+
|
|
|
+ if isinstance(io, str) and not any(
|
|
|
+ [
|
|
|
+ is_file_like(io),
|
|
|
+ file_exists(io),
|
|
|
+ is_url(io),
|
|
|
+ is_fsspec_url(io),
|
|
|
+ ]
|
|
|
+ ):
|
|
|
+ warnings.warn(
|
|
|
+ "Passing literal html to 'read_html' is deprecated and "
|
|
|
+ "will be removed in a future version. To read from a "
|
|
|
+ "literal string, wrap it in a 'StringIO' object.",
|
|
|
+ FutureWarning,
|
|
|
+ stacklevel=find_stack_level(),
|
|
|
+ )
|
|
|
+
|
|
|
+ return _parse(
|
|
|
+ flavor=flavor,
|
|
|
+ io=io,
|
|
|
+ match=match,
|
|
|
+ header=header,
|
|
|
+ index_col=index_col,
|
|
|
+ skiprows=skiprows,
|
|
|
+ parse_dates=parse_dates,
|
|
|
+ thousands=thousands,
|
|
|
+ attrs=attrs,
|
|
|
+ encoding=encoding,
|
|
|
+ decimal=decimal,
|
|
|
+ converters=converters,
|
|
|
+ na_values=na_values,
|
|
|
+ keep_default_na=keep_default_na,
|
|
|
+ displayed_only=displayed_only,
|
|
|
+ extract_links=extract_links,
|
|
|
+ dtype_backend=dtype_backend,
|
|
|
+ storage_options=storage_options,
|
|
|
+ custom_args=custom_args,
|
|
|
+ )
|
