# -*- coding: utf-8 -*-
"""PyShark Support
=====================
.. module:: pcapkit.foundation.engines.pyshark
This module contains the implementation for `PyShark`_ engine
support, as is used by :class:`pcapkit.foundation.extraction.Extractor`.
"""
import sys
from typing import TYPE_CHECKING, cast
from pcapkit.foundation.engines.engine import EngineBase
from pcapkit.foundation.reassembly import ReassemblyManager
from pcapkit.utilities.exceptions import stacklevel
from pcapkit.utilities.logging import get_logger
from pcapkit.utilities.warnings import AttributeWarning, warn
__all__ = ['PyShark']
if TYPE_CHECKING:
from typing import Any, Optional, Protocol, Type
from pyshark.capture.file_capture import FileCapture as PySharkFileCapture
from pyshark.packet.layers.base import BaseLayer
from pyshark.packet.packet import Packet as PySharkPacket
from pcapkit.foundation.extraction import Extractor
class _PySharkPackage(Protocol):
"""|pyshark|_, as :class:`PyShark` reads it through :attr:`PyShark._expkg` (:issue:`1878`)."""
@property
def FileCapture(self) -> 'Type[PySharkFileCapture]': ...
#: logging.Logger: Module-level logger, a child of the package-wide
#: :data:`pcapkit.utilities.logging.logger`.
logger = get_logger(__name__)
def _frame_layer(packet: 'PySharkPacket') -> 'Optional[BaseLayer]':
"""The ``frame`` layer of a `PyShark`_ packet, or :data:`None` when the record has none.
Looked up by name, because `PyShark`_ takes the second PDML ``<proto>`` as
``frame_info``. On a packet with a comment, that is ``pkt_comment``, which has
no ``protocols``, and the frame layer is ``layers[0]``. By its source,
:program:`tshark` 4.2.2 roots a Sysdig event at ``syscall`` and a Netflix
custom block at ``bblog``, so those records have no ``frame`` layer.
"""
for layer in (packet.frame_info, *packet.layers):
if layer.layer_name == 'frame':
return layer
return None
def _is_packet(packet: 'PySharkPacket') -> 'bool':
"""Whether a `PyShark`_ packet is a packet record, rather than another kind of tshark record.
:program:`tshark` numbers every record it reads, not only packets: a PCAP-NG
systemd Journal Export Block, Custom Block or Sysdig Event Block is a frame
to it, measured with :program:`tshark` 4.6.9. This engine counts packet
records only.
The marker is ``frame.encap_type`` on the record's :func:`_frame_layer`.
Wireshark adds it for ``REC_TYPE_PACKET`` records only, in ``dissect_frame``
of :file:`epan/dissectors/packet-frame.c`, in 4.2.2 and 4.6.9 alike. The
summary line is not used, because it differs between those versions. A
record with no ``frame`` layer is not a packet either.
"""
frame = _frame_layer(packet)
return frame is not None and 'encap_type' in frame.field_names
def _lost_tsharks(capture: 'PySharkFileCapture', error: 'Exception') -> 'Optional[set[Any]]':
"""The :program:`tshark` processes of ``capture`` whose exit status asyncio lost.
:data:`None` unless ``error`` is the crash `PyShark`_ reported for them. See
:meth:`PyShark.close` for the condition, and why it is enough.
"""
from pyshark.capture.capture import TSharkCrashException # isort:skip
running = getattr(capture, '_running_processes', None)
if not (isinstance(error, TSharkCrashException) and isinstance(running, set) and running
and getattr(capture, '_eof_reached', None) is True):
return None
for process in running:
extra = getattr(getattr(process, '_transport', None), 'get_extra_info', None)
popen = None if extra is None else extra('subprocess')
if process.returncode != 255 or getattr(popen, 'returncode', None) != 0:
return None
return running
[docs]
class PyShark(EngineBase['PySharkPacket']):
"""PyShark engine support.
Args:
extractor: :class:`~pcapkit.foundation.extraction.Extractor` instance.
"""
if TYPE_CHECKING:
#: Engine extraction package, i.e. |pyshark|_.
_expkg: '_PySharkPackage'
#: Engine extraction temporary storage.
_extmp: 'PySharkFileCapture'
##########################################################################
# Defaults.
##########################################################################
#: Engine name.
__engine_name__ = 'PyShark'
#: Engine module name.
__engine_module__ = 'pyshark'
#: First Python version `PyShark`_ does not work on, as a ``(major, minor)``
#: pair. Released 0.6 builds its event loop with
#: ``asyncio.get_event_loop_policy().get_event_loop()``, and Python 3.14 made
#: :func:`asyncio.get_event_loop` raise :exc:`RuntimeError` when no current
#: event loop exists instead of quietly creating one.
PYTHON_CEILING = (3, 14)
##########################################################################
# Class methods.
##########################################################################
[docs]
@classmethod
def unsupported_reason(cls) -> 'Optional[str]':
"""Why this engine cannot run here, or :data:`None` when it can.
Consulted when :class:`~pcapkit.foundation.extraction.Extractor` settles
its engine, *before* the import test, because nothing that stops this
engine is visible to an import. `PyShark`_ imports fine and fails when
used, which without this hook escapes from :meth:`run` as a hard error
rather than degrading to the default engine with a warning.
**The interpreter.** |pyshark|_ 0.6 calls
``asyncio.get_event_loop_policy().get_event_loop()`` in
``Capture._setup_eventloop``, in a fresh interpreter with no
running loop. Measured: 3.10 and 3.11 return a loop
silently, 3.12 returns one with a :exc:`DeprecationWarning`, and 3.14
raises ``RuntimeError: There is no current event loop in thread
'MainThread'``; on 3.13 |pyshark|_ extracted a capture. Hence
:attr:`PYTHON_CEILING` is ``(3, 14)``.
**The** :program:`tshark` **binary.** |pyshark|_ wraps Wireshark's
command-line tool and parses nothing itself. The check asks |pyshark|_'s
own ``get_process_path()`` rather than :func:`shutil.which`, which would
refuse setups that work: |pyshark|_ reads ``tshark_path`` from its
:file:`config.ini` first, then :envvar:`PATH` on POSIX, both Program Files
directories on Windows, and :file:`/Applications/Wireshark.app` on macOS.
Note:
Deliberately **not cached**: it runs per
:class:`~pcapkit.foundation.extraction.Extractor` rather than per
frame, far below the cost of opening the capture, and caching would
keep a stale answer after Wireshark is installed or :envvar:`PATH` is
fixed.
Returns:
A short phrase naming the limitation, or :data:`None`.
"""
if sys.version_info[:2] >= cls.PYTHON_CEILING:
return (f'pyshark does not support Python '
f'{sys.version_info[0]}.{sys.version_info[1]}; it builds its event '
'loop with `asyncio.get_event_loop_policy().get_event_loop()`, which '
'raises RuntimeError '
'from Python 3.14 when no current event loop exists')
try:
from pyshark.tshark.tshark import get_process_path # isort:skip
except ImportError:
# Not installed, which is ``Extractor.import_test``'s business -- it
# reports that case in its own words, and answering here as well would
# produce two warnings for one problem. An ImportError from a *renamed*
# upstream helper lands here too, and the engine then simply proceeds
# without the ``tshark`` check.
return None
try:
get_process_path()
except Exception as exc: # pylint: disable=broad-except
# ``TSharkNotFoundException`` by name, but caught broadly: the whole
# point is to turn any failure to locate the binary into a reason
# rather than let it escape, and upstream is free to raise something
# else. Its own message lists every path it searched, which is exactly
# what the user needs, so it is quoted rather than summarised.
return (f'pyshark requires Wireshark\'s `tshark` binary, which pyshark '
f'could not find -- {exc}')
return None
##########################################################################
# Data models.
##########################################################################
def __init__(self, extractor: 'Extractor') -> 'None':
import pyshark # isort:skip
self._expkg = pyshark
self._extmp = cast('PySharkFileCapture', None)
super().__init__(extractor)
##########################################################################
# Methods.
##########################################################################
[docs]
def run(self) -> 'None':
"""Call ``pyshark.FileCapture`` to extract PCAP files.
This method assigns :attr:`self._expkg <PyShark._expkg>`
as |pyshark|_ and :attr:`self._extmp <PyShark._extmp>`
as an iterator from ``pyshark.FileCapture``.
The global header is parsed and dumped first, by
:meth:`self.extractor.record_header <pcapkit.foundation.extraction.Extractor.record_header>`.
Warns:
AttributeWarning: Warns under the following circumstances:
* if :attr:`self.extractor._exlyr <pcapkit.foundation.extraction.Extractor._exlyr>`
and/or :attr:`self.extractor._exptl <pcapkit.foundation.extraction.Extractor._exptl>`
is provided as the PyShark engine does not
support such operations.
* if reassembly is enabled, as the PyShark engine
does not support such operation.
* if :attr:`self.extractor._exctx <pcapkit.foundation.extraction.Extractor._exctx>`
is provided, as the PyShark engine does not parse with
:mod:`pcapkit`'s own protocol implementations.
"""
ext = self._extractor
if ext._exlyr != 'none' or ext._exptl != 'null':
warn("'Extractor(engine=pyshark)' does not support protocol and layer threshold; "
f"'layer={ext._exlyr}' and 'protocol={ext._exptl}' ignored",
AttributeWarning, stacklevel=stacklevel())
if ext._exctx:
warn("'Extractor(engine=pyshark)' does not parse with pcapkit's own protocol "
"implementations, so the parsing context supplied through "
"'context=' is ignored",
AttributeWarning, stacklevel=stacklevel())
if ext._flag_r and (ext._ipv4 or ext._ipv6 or ext._tcp or ext._dtls):
ext._flag_r = False
logger.debug('pyshark: reassembly unsupported, disabling it')
ext._reasm = ReassemblyManager(ipv4=None, ipv6=None, tcp=None, dtls=None)
warn("'Extractor(engine=pyshark)' object does not support reassembly; "
f"so 'ipv4={ext._ipv4}', 'ipv6={ext._ipv6}', 'tcp={ext._tcp}' and "
f"'dtls={ext._dtls}' will be ignored",
AttributeWarning, stacklevel=stacklevel())
# setup verbose handler, unless the caller gave one of their own
# (#1874); a packet always has a frame layer, since read_frame skips
# every record that does not (#1515)
if ext._vfunc_default:
ext._vfunc = lambda e, f: print(
f'Frame {e._frnum:>3d}: ' # pylint: disable=protected-access
f'{cast("BaseLayer", _frame_layer(cast("PySharkPacket", f))).protocols}'
)
# extract global header
ext.record_header()
# extract & analyse file
logger.debug('pyshark: opening %s', ext._ifnm)
self._extmp = self._expkg.FileCapture(ext._ifnm, keep_packets=False)
[docs]
def read_frame(self) -> 'PySharkPacket':
"""Read frames with PyShark engine.
A tshark record that is not a packet, such as a PCAP-NG systemd Journal
Export Block, is skipped. Frames are numbered by count, not by tshark's
frame number.
Returns:
Parsed frame instance.
See Also:
Please refer to :meth:`PCAP.read_frame <pcapkit.foundation.engines.pcap.PCAP.read_frame>`
for more operational information.
"""
from pcapkit.toolkit.pyshark import packet2dict, tcp_traceflow
ext = self._extractor
# fetch the next PyShark packet that is a packet record
while True:
packet = self._extmp.next()
if _is_packet(packet):
break
logger.debug('pyshark: skipping tshark frame %s, not a packet record', packet.number)
# verbose output
ext._frnum += 1
ext._vfunc(ext, packet)
# write output
frnum = f'Frame {ext._frnum}'
if not ext._flag_q:
info = packet2dict(packet)
if ext._flag_f:
ofile = ext._ofile(f'{ext._ofnm}/{frnum}.{ext._fext}')
ofile(info, name=frnum)
else:
ext._ofile(info, name=frnum)
# trace flows
if ext._flag_t:
if ext._tcp:
data_tf_tcp = tcp_traceflow(packet, count=ext._frnum)
if data_tf_tcp is not None:
ext._trace.tcp(data_tf_tcp)
# record frames
if ext._flag_d:
# setattr(packet, 'packet2dict', packet2dict)
ext._frame.append(packet)
# return frame record
return packet
[docs]
def close(self) -> 'None':
"""Close engine.
This method is to be used for closing the engine instance. It is to
close the engine instance after the extraction process is finished.
A no-op if :meth:`run` never opened the capture, which is the case when
the :class:`~pcapkit.foundation.extraction.Extractor` constructor raises
before it (:issue:`2107`).
`PyShark`_ can raise ``TSharkCrashException`` with ``retcode: 255`` here
for a :program:`tshark` that exited cleanly (:issue:`2633`). When a
:meth:`subprocess.Popen.poll` reaps a :program:`tshark` that exited before
the event loop handled its ``SIGCHLD``, the child watcher then finds no
child and reports ``255``. Its cleanup kills :program:`tshark` through
:mod:`asyncio`, whose ``kill`` on CPython 3.9 to 3.12 goes through
:meth:`subprocess.Popen.send_signal`, which polls first (CPython gh-87744).
That is background, not a gate: the check below reads the evidence the
loss leaves, on every Python.
The ``Popen`` keeps the status its poll read, so that crash is logged at
:data:`logging.DEBUG` instead of raised, but only when the capture was read
to its end and every :program:`tshark` it still lists reports ``255`` to
:mod:`asyncio` and ``0`` to its ``Popen``. A :program:`tshark` that failed,
or exited ``255`` itself, keeps that status there and still raises, as
does any other error. The one case this cannot tell apart is a
:program:`tshark` exiting ``255`` that the child watcher reaps just before
the poll, which then finds no child and records ``0``.
"""
if self._extmp is not None:
try:
self._extmp.close()
except Exception as error: # pylint: disable=broad-except
lost = _lost_tsharks(self._extmp, error)
if lost is None:
raise
logger.debug('pyshark: not raising %s for tshark (pid %s) at end of file: asyncio '
'reports 255, but its Popen read 0, so asyncio lost the exit status',
type(error).__name__, ', '.join(str(process.pid) for process in lost))
# NOTE: still listed, the reaped tshark makes the capture's ``__del__`` raise
# this again; closing once more runs the rest of pyshark's own close
lost.clear()
self._extmp.close()