|
""" |
|
Tools to open .py files as Unicode, using the encoding specified within the file, |
|
as per PEP 263. |
|
|
|
Much of the code is taken from the tokenize module in Python 3.2. |
|
""" |
|
|
|
import io |
|
from io import TextIOWrapper, BytesIO |
|
from pathlib import Path |
|
import re |
|
from tokenize import open, detect_encoding |
|
|
|
cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)", re.UNICODE) |
|
cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE) |
|
|
|
def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True): |
|
"""Converts a bytes string with python source code to unicode. |
|
|
|
Unicode strings are passed through unchanged. Byte strings are checked |
|
for the python source file encoding cookie to determine encoding. |
|
txt can be either a bytes buffer or a string containing the source |
|
code. |
|
""" |
|
if isinstance(txt, str): |
|
return txt |
|
if isinstance(txt, bytes): |
|
buffer = BytesIO(txt) |
|
else: |
|
buffer = txt |
|
try: |
|
encoding, _ = detect_encoding(buffer.readline) |
|
except SyntaxError: |
|
encoding = "ascii" |
|
buffer.seek(0) |
|
with TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) as text: |
|
text.mode = 'r' |
|
if skip_encoding_cookie: |
|
return u"".join(strip_encoding_cookie(text)) |
|
else: |
|
return text.read() |
|
|
|
def strip_encoding_cookie(filelike): |
|
"""Generator to pull lines from a text-mode file, skipping the encoding |
|
cookie if it is found in the first two lines. |
|
""" |
|
it = iter(filelike) |
|
try: |
|
first = next(it) |
|
if not cookie_comment_re.match(first): |
|
yield first |
|
second = next(it) |
|
if not cookie_comment_re.match(second): |
|
yield second |
|
except StopIteration: |
|
return |
|
|
|
for line in it: |
|
yield line |
|
|
|
def read_py_file(filename, skip_encoding_cookie=True): |
|
"""Read a Python file, using the encoding declared inside the file. |
|
|
|
Parameters |
|
---------- |
|
filename : str |
|
The path to the file to read. |
|
skip_encoding_cookie : bool |
|
If True (the default), and the encoding declaration is found in the first |
|
two lines, that line will be excluded from the output. |
|
|
|
Returns |
|
------- |
|
A unicode string containing the contents of the file. |
|
""" |
|
filepath = Path(filename) |
|
with open(filepath) as f: |
|
if skip_encoding_cookie: |
|
return "".join(strip_encoding_cookie(f)) |
|
else: |
|
return f.read() |
|
|
|
def read_py_url(url, errors='replace', skip_encoding_cookie=True): |
|
"""Read a Python file from a URL, using the encoding declared inside the file. |
|
|
|
Parameters |
|
---------- |
|
url : str |
|
The URL from which to fetch the file. |
|
errors : str |
|
How to handle decoding errors in the file. Options are the same as for |
|
bytes.decode(), but here 'replace' is the default. |
|
skip_encoding_cookie : bool |
|
If True (the default), and the encoding declaration is found in the first |
|
two lines, that line will be excluded from the output. |
|
|
|
Returns |
|
------- |
|
A unicode string containing the contents of the file. |
|
""" |
|
|
|
from urllib.request import urlopen |
|
response = urlopen(url) |
|
buffer = io.BytesIO(response.read()) |
|
return source_to_unicode(buffer, errors, skip_encoding_cookie) |
|
|
