chore: commit mileage updates and accumulated project artifacts

This commit is contained in:
lingniu
2026-09-21 09:15:54 +08:00
parent 6cc872ceb6
commit 8b32ae2988
767 changed files with 572885 additions and 17 deletions
+410
View File
@@ -0,0 +1,410 @@
#!/Users/lingniu/.cache/codex-runtimes/codex-primary-runtime/dependencies/python/bin/python3
# Copyright (c) 2005-2012 Stephen John Machin, Lingfo Pty Ltd
# This script is part of the xlrd package, which is released under a
# BSD-style licence.
from __future__ import print_function
cmd_doc = """
Commands:
2rows Print the contents of first and last row in each sheet
3rows Print the contents of first, second and last row in each sheet
bench Same as "show", but doesn't print -- for profiling
biff_count[1] Print a count of each type of BIFF record in the file
biff_dump[1] Print a dump (char and hex) of the BIFF records in the file
fonts hdr + print a dump of all font objects
hdr Mini-overview of file (no per-sheet information)
hotshot Do a hotshot profile run e.g. ... -f1 hotshot bench bigfile*.xls
labels Dump of sheet.col_label_ranges and ...row... for each sheet
name_dump Dump of each object in book.name_obj_list
names Print brief information for each NAME record
ov Overview of file
profile Like "hotshot", but uses cProfile
show Print the contents of all rows in each sheet
version[0] Print versions of xlrd and Python and exit
xfc Print "XF counts" and cell-type counts -- see code for details
[0] means no file arg
[1] means only one file arg i.e. no glob.glob pattern
"""
options = None
if __name__ == "__main__":
import xlrd
import sys
import time
import glob
import traceback
import gc
from xlrd.timemachine import xrange, REPR
class LogHandler(object):
def __init__(self, logfileobj):
self.logfileobj = logfileobj
self.fileheading = None
self.shown = 0
def setfileheading(self, fileheading):
self.fileheading = fileheading
self.shown = 0
def write(self, text):
if self.fileheading and not self.shown:
self.logfileobj.write(self.fileheading)
self.shown = 1
self.logfileobj.write(text)
null_cell = xlrd.empty_cell
def show_row(bk, sh, rowx, colrange, printit):
if bk.ragged_rows:
colrange = range(sh.row_len(rowx))
if not colrange: return
if printit: print()
if bk.formatting_info:
for colx, ty, val, cxfx in get_row_data(bk, sh, rowx, colrange):
if printit:
print("cell %s%d: type=%d, data: %r, xfx: %s"
% (xlrd.colname(colx), rowx+1, ty, val, cxfx))
else:
for colx, ty, val, _unused in get_row_data(bk, sh, rowx, colrange):
if printit:
print("cell %s%d: type=%d, data: %r" % (xlrd.colname(colx), rowx+1, ty, val))
def get_row_data(bk, sh, rowx, colrange):
result = []
dmode = bk.datemode
ctys = sh.row_types(rowx)
cvals = sh.row_values(rowx)
for colx in colrange:
cty = ctys[colx]
cval = cvals[colx]
if bk.formatting_info:
cxfx = str(sh.cell_xf_index(rowx, colx))
else:
cxfx = ''
if cty == xlrd.XL_CELL_DATE:
try:
showval = xlrd.xldate_as_tuple(cval, dmode)
except xlrd.XLDateError as e:
showval = "%s:%s" % (type(e).__name__, e)
cty = xlrd.XL_CELL_ERROR
elif cty == xlrd.XL_CELL_ERROR:
showval = xlrd.error_text_from_code.get(cval, '<Unknown error code 0x%02x>' % cval)
else:
showval = cval
result.append((colx, cty, showval, cxfx))
return result
def bk_header(bk):
print()
print("BIFF version: %s; datemode: %s"
% (xlrd.biff_text_from_num[bk.biff_version], bk.datemode))
print("codepage: %r (encoding: %s); countries: %r"
% (bk.codepage, bk.encoding, bk.countries))
print("Last saved by: %r" % bk.user_name)
print("Number of data sheets: %d" % bk.nsheets)
print("Use mmap: %d; Formatting: %d; On demand: %d"
% (bk.use_mmap, bk.formatting_info, bk.on_demand))
print("Ragged rows: %d" % bk.ragged_rows)
if bk.formatting_info:
print("FORMATs: %d, FONTs: %d, XFs: %d"
% (len(bk.format_list), len(bk.font_list), len(bk.xf_list)))
if not options.suppress_timing:
print("Load time: %.2f seconds (stage 1) %.2f seconds (stage 2)"
% (bk.load_time_stage_1, bk.load_time_stage_2))
print()
def show_fonts(bk):
print("Fonts:")
for x in xrange(len(bk.font_list)):
font = bk.font_list[x]
font.dump(header='== Index %d ==' % x, indent=4)
def show_names(bk, dump=0):
bk_header(bk)
if bk.biff_version < 50:
print("Names not extracted in this BIFF version")
return
nlist = bk.name_obj_list
print("Name list: %d entries" % len(nlist))
for nobj in nlist:
if dump:
nobj.dump(sys.stdout,
header="\n=== Dump of name_obj_list[%d] ===" % nobj.name_index)
else:
print("[%d]\tName:%r macro:%r scope:%d\n\tresult:%r\n"
% (nobj.name_index, nobj.name, nobj.macro, nobj.scope, nobj.result))
def print_labels(sh, labs, title):
if not labs:return
for rlo, rhi, clo, chi in labs:
print("%s label range %s:%s contains:"
% (title, xlrd.cellname(rlo, clo), xlrd.cellname(rhi-1, chi-1)))
for rx in xrange(rlo, rhi):
for cx in xrange(clo, chi):
print(" %s: %r" % (xlrd.cellname(rx, cx), sh.cell_value(rx, cx)))
def show_labels(bk):
# bk_header(bk)
hdr = 0
for shx in range(bk.nsheets):
sh = bk.sheet_by_index(shx)
clabs = sh.col_label_ranges
rlabs = sh.row_label_ranges
if clabs or rlabs:
if not hdr:
bk_header(bk)
hdr = 1
print("sheet %d: name = %r; nrows = %d; ncols = %d" %
(shx, sh.name, sh.nrows, sh.ncols))
print_labels(sh, clabs, 'Col')
print_labels(sh, rlabs, 'Row')
if bk.on_demand: bk.unload_sheet(shx)
def show(bk, nshow=65535, printit=1):
bk_header(bk)
if 0:
rclist = xlrd.sheet.rc_stats.items()
rclist = sorted(rclist)
print("rc stats")
for k, v in rclist:
print("0x%04x %7d" % (k, v))
if options.onesheet:
try:
shx = int(options.onesheet)
except ValueError:
shx = bk.sheet_by_name(options.onesheet).number
shxrange = [shx]
else:
shxrange = range(bk.nsheets)
# print("shxrange", list(shxrange))
for shx in shxrange:
sh = bk.sheet_by_index(shx)
nrows, ncols = sh.nrows, sh.ncols
colrange = range(ncols)
anshow = min(nshow, nrows)
print("sheet %d: name = %s; nrows = %d; ncols = %d" %
(shx, REPR(sh.name), sh.nrows, sh.ncols))
if nrows and ncols:
# Beat the bounds
for rowx in xrange(nrows):
nc = sh.row_len(rowx)
if nc:
sh.row_types(rowx)[nc-1]
sh.row_values(rowx)[nc-1]
sh.cell(rowx, nc-1)
for rowx in xrange(anshow-1):
if not printit and rowx % 10000 == 1 and rowx > 1:
print("done %d rows" % (rowx-1,))
show_row(bk, sh, rowx, colrange, printit)
if anshow and nrows:
show_row(bk, sh, nrows-1, colrange, printit)
print()
if bk.on_demand: bk.unload_sheet(shx)
def count_xfs(bk):
bk_header(bk)
for shx in range(bk.nsheets):
sh = bk.sheet_by_index(shx)
nrows = sh.nrows
print("sheet %d: name = %r; nrows = %d; ncols = %d" %
(shx, sh.name, sh.nrows, sh.ncols))
# Access all xfindexes to force gathering stats
type_stats = [0, 0, 0, 0, 0, 0, 0]
for rowx in xrange(nrows):
for colx in xrange(sh.row_len(rowx)):
xfx = sh.cell_xf_index(rowx, colx)
assert xfx >= 0
cty = sh.cell_type(rowx, colx)
type_stats[cty] += 1
print("XF stats", sh._xf_index_stats)
print("type stats", type_stats)
print()
if bk.on_demand: bk.unload_sheet(shx)
def main(cmd_args):
import optparse
global options
usage = "\n%prog [options] command [input-file-patterns]\n" + cmd_doc
oparser = optparse.OptionParser(usage)
oparser.add_option(
"-l", "--logfilename",
default="",
help="contains error messages")
oparser.add_option(
"-v", "--verbosity",
type="int", default=0,
help="level of information and diagnostics provided")
oparser.add_option(
"-m", "--mmap",
type="int", default=-1,
help="1: use mmap; 0: don't use mmap; -1: accept heuristic")
oparser.add_option(
"-e", "--encoding",
default="",
help="encoding override")
oparser.add_option(
"-f", "--formatting",
type="int", default=0,
help="0 (default): no fmt info\n"
"1: fmt info (all cells)\n",
)
oparser.add_option(
"-g", "--gc",
type="int", default=0,
help="0: auto gc enabled; 1: auto gc disabled, manual collect after each file; 2: no gc")
oparser.add_option(
"-s", "--onesheet",
default="",
help="restrict output to this sheet (name or index)")
oparser.add_option(
"-u", "--unnumbered",
action="store_true", default=0,
help="omit line numbers or offsets in biff_dump")
oparser.add_option(
"-d", "--on-demand",
action="store_true", default=0,
help="load sheets on demand instead of all at once")
oparser.add_option(
"-t", "--suppress-timing",
action="store_true", default=0,
help="don't print timings (diffs are less messy)")
oparser.add_option(
"-r", "--ragged-rows",
action="store_true", default=0,
help="open_workbook(..., ragged_rows=True)")
options, args = oparser.parse_args(cmd_args)
if len(args) == 1 and args[0] in ("version", ):
pass
elif len(args) < 2:
oparser.error("Expected at least 2 args, found %d" % len(args))
cmd = args[0]
xlrd_version = getattr(xlrd, "__VERSION__", "unknown; before 0.5")
if cmd == 'biff_dump':
xlrd.dump(args[1], unnumbered=options.unnumbered)
sys.exit(0)
if cmd == 'biff_count':
xlrd.count_records(args[1])
sys.exit(0)
if cmd == 'version':
print("xlrd: %s, from %s" % (xlrd_version, xlrd.__file__))
print("Python:", sys.version)
sys.exit(0)
if options.logfilename:
logfile = LogHandler(open(options.logfilename, 'w'))
else:
logfile = sys.stdout
mmap_opt = options.mmap
mmap_arg = xlrd.USE_MMAP
if mmap_opt in (1, 0):
mmap_arg = mmap_opt
elif mmap_opt != -1:
print('Unexpected value (%r) for mmap option -- assuming default' % mmap_opt)
fmt_opt = options.formatting | (cmd in ('xfc', ))
gc_mode = options.gc
if gc_mode:
gc.disable()
for pattern in args[1:]:
for fname in glob.glob(pattern):
print("\n=== File: %s ===" % fname)
if logfile != sys.stdout:
logfile.setfileheading("\n=== File: %s ===\n" % fname)
if gc_mode == 1:
n_unreachable = gc.collect()
if n_unreachable:
print("GC before open:", n_unreachable, "unreachable objects")
try:
t0 = time.time()
bk = xlrd.open_workbook(
fname,
verbosity=options.verbosity, logfile=logfile,
use_mmap=mmap_arg,
encoding_override=options.encoding,
formatting_info=fmt_opt,
on_demand=options.on_demand,
ragged_rows=options.ragged_rows,
)
t1 = time.time()
if not options.suppress_timing:
print("Open took %.2f seconds" % (t1-t0,))
except xlrd.XLRDError as e:
print("*** Open failed: %s: %s" % (type(e).__name__, e))
continue
except KeyboardInterrupt:
print("*** KeyboardInterrupt ***")
traceback.print_exc(file=sys.stdout)
sys.exit(1)
except BaseException as e:
print("*** Open failed: %s: %s" % (type(e).__name__, e))
traceback.print_exc(file=sys.stdout)
continue
t0 = time.time()
if cmd == 'hdr':
bk_header(bk)
elif cmd == 'ov': # OverView
show(bk, 0)
elif cmd == 'show': # all rows
show(bk)
elif cmd == '2rows': # first row and last row
show(bk, 2)
elif cmd == '3rows': # first row, 2nd row and last row
show(bk, 3)
elif cmd == 'bench':
show(bk, printit=0)
elif cmd == 'fonts':
bk_header(bk)
show_fonts(bk)
elif cmd == 'names': # named reference list
show_names(bk)
elif cmd == 'name_dump': # named reference list
show_names(bk, dump=1)
elif cmd == 'labels':
show_labels(bk)
elif cmd == 'xfc':
count_xfs(bk)
else:
print("*** Unknown command <%s>" % cmd)
sys.exit(1)
del bk
if gc_mode == 1:
n_unreachable = gc.collect()
if n_unreachable:
print("GC post cmd:", fname, "->", n_unreachable, "unreachable objects")
if not options.suppress_timing:
t1 = time.time()
print("\ncommand took %.2f seconds\n" % (t1-t0,))
return None
av = sys.argv[1:]
if not av:
main(av)
firstarg = av[0].lower()
if firstarg == "hotshot":
import hotshot
import hotshot.stats
av = av[1:]
prof_log_name = "XXXX.prof"
prof = hotshot.Profile(prof_log_name)
# benchtime, result = prof.runcall(main, *av)
result = prof.runcall(main, *(av, ))
print("result", repr(result))
prof.close()
stats = hotshot.stats.load(prof_log_name)
stats.strip_dirs()
stats.sort_stats('time', 'calls')
stats.print_stats(20)
elif firstarg == "profile":
import cProfile
av = av[1:]
cProfile.run('main(av)', 'YYYY.prof')
import pstats
p = pstats.Stats('YYYY.prof')
p.strip_dirs().sort_stats('cumulative').print_stats(30)
else:
main(av)
@@ -0,0 +1,131 @@
Metadata-Version: 2.4
Name: PyMySQL
Version: 1.2.0
Summary: Pure Python MySQL Driver
Author-email: Inada Naoki <songofacandy@gmail.com>, Yutaka Matsubara <yutaka.matsubara@gmail.com>
License-Expression: MIT
Project-URL: Project, https://github.com/PyMySQL/PyMySQL
Project-URL: Documentation, https://pymysql.readthedocs.io/
Project-URL: Changelog, https://github.com/PyMySQL/PyMySQL/blob/main/CHANGELOG.md
Keywords: MySQL
Classifier: Development Status :: 5 - Production/Stable
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Intended Audience :: Developers
Classifier: Topic :: Database
Requires-Python: >=3.9
Description-Content-Type: text/markdown
License-File: LICENSE
Provides-Extra: rsa
Requires-Dist: cryptography>=46.0.7; extra == "rsa"
Provides-Extra: ed25519
Requires-Dist: PyNaCl>=1.6.2; extra == "ed25519"
Dynamic: license-file
[![Documentation Status](https://readthedocs.org/projects/pymysql/badge/?version=latest)](https://pymysql.readthedocs.io/)
[![codecov](https://codecov.io/gh/PyMySQL/PyMySQL/branch/main/graph/badge.svg?token=ppEuaNXBW4)](https://codecov.io/gh/PyMySQL/PyMySQL)
[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/PyMySQL/PyMySQL)
# PyMySQL
This package contains a pure-Python MySQL and MariaDB client library, based on
[PEP 249](https://www.python.org/dev/peps/pep-0249/).
## Requirements
- Python -- one of the following:
- [CPython](https://www.python.org/) : 3.9 and newer
- [PyPy](https://pypy.org/) : Latest 3.x version
- MySQL Server -- one of the following:
- [MySQL](https://www.mysql.com/) LTS versions
- [MariaDB](https://mariadb.org/) LTS versions
## Installation
Package is uploaded on [PyPI](https://pypi.org/project/PyMySQL).
You can install it with pip:
$ python3 -m pip install PyMySQL
To use "sha256_password" or "caching_sha2_password" for authenticate,
you need to install additional dependency:
$ python3 -m pip install PyMySQL[rsa]
To use MariaDB's "ed25519" authentication method, you need to install
additional dependency:
$ python3 -m pip install PyMySQL[ed25519]
## Documentation
Documentation is available online: <https://pymysql.readthedocs.io/>
For support, please refer to the
[StackOverflow](https://stackoverflow.com/questions/tagged/pymysql).
## Example
The following examples make use of a simple table
``` sql
CREATE TABLE `users` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`email` varchar(255) COLLATE utf8_bin NOT NULL,
`password` varchar(255) COLLATE utf8_bin NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin
AUTO_INCREMENT=1 ;
```
``` python
import pymysql.cursors
# Connect to the database
connection = pymysql.connect(host='localhost',
user='user',
password='passwd',
database='db',
cursorclass=pymysql.cursors.DictCursor)
with connection:
with connection.cursor() as cursor:
# Create a new record
sql = "INSERT INTO `users` (`email`, `password`) VALUES (%s, %s)"
cursor.execute(sql, ('webmaster@python.org', 'very-secret'))
# connection is not autocommit by default. So you must commit to save
# your changes.
connection.commit()
with connection.cursor() as cursor:
# Read a single record
sql = "SELECT `id`, `password` FROM `users` WHERE `email`=%s"
cursor.execute(sql, ('webmaster@python.org',))
result = cursor.fetchone()
print(result)
```
This example will print:
``` python
{'password': 'very-secret', 'id': 1}
```
## Resources
- DB-API 2.0: <https://www.python.org/dev/peps/pep-0249/>
- MySQL Reference Manuals: <https://dev.mysql.com/doc/>
- Getting Help With MariaDB <https://mariadb.com/kb/en/getting-help-with-mariadb/>
- MySQL client/server protocol:
<https://dev.mysql.com/doc/internals/en/client-server-protocol.html>
- "Connector" channel in MySQL Community Slack:
<https://lefred.be/mysql-community-on-slack/>
- PyMySQL mailing list:
<https://groups.google.com/forum/#!forum/pymysql-users>
## License
PyMySQL is released under the MIT License. See LICENSE for more
information.
@@ -0,0 +1,43 @@
pymysql-1.2.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
pymysql-1.2.0.dist-info/METADATA,sha256=oyVsiVIGcl45MPHJ-fxObPZmLJkzxJh1nA7ZkiUM9X8,4306
pymysql-1.2.0.dist-info/RECORD,,
pymysql-1.2.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
pymysql-1.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
pymysql-1.2.0.dist-info/licenses/LICENSE,sha256=MUEg3GXwgA9ziksxQAx27hTezR--d86cNUCkIbhup7Y,1070
pymysql-1.2.0.dist-info/top_level.txt,sha256=IKlV-f4o90sOdnMd6HBvo0l2nqfJOGUzkwZeaEEGuRg,8
pymysql/__init__.py,sha256=zNamwe3r_VjTlS-z0bqlpNT4clGPjSLeKhfG8Q3fZZ8,4262
pymysql/__pycache__/__init__.cpython-312.pyc,,
pymysql/__pycache__/_auth.cpython-312.pyc,,
pymysql/__pycache__/charset.cpython-312.pyc,,
pymysql/__pycache__/connections.cpython-312.pyc,,
pymysql/__pycache__/converters.cpython-312.pyc,,
pymysql/__pycache__/cursors.cpython-312.pyc,,
pymysql/__pycache__/err.cpython-312.pyc,,
pymysql/__pycache__/optionfile.cpython-312.pyc,,
pymysql/__pycache__/protocol.cpython-312.pyc,,
pymysql/__pycache__/times.cpython-312.pyc,,
pymysql/_auth.py,sha256=7bIFnJ7lJrFEhKLEnHGo1-h7E5cnZB2211KE1vatBAQ,7638
pymysql/charset.py,sha256=Y4GgMDxn0Yz-99NwstfCLeCfoRFdwywWoHrn5Gnvghk,10258
pymysql/connections.py,sha256=ZJux4MPj55eyts1Ko1sghO3XXEsvbFsZAK5pYoW5rM4,55738
pymysql/constants/CLIENT.py,sha256=SSvMFPZCTVMU1UWa4zOrfhYMDdR2wG2mS0E5GzJhDsg,878
pymysql/constants/COMMAND.py,sha256=TGITAUcNWlq2Gwg2wv5UK2ykdTd4LYTk_EcJJOCpGIc,679
pymysql/constants/CR.py,sha256=Qk35FWRMxRHd6Sa9CCIATMh7jegR3xnLdrdaBCT0dTQ,2320
pymysql/constants/ER.py,sha256=nwqX_r0o4mmN4Cxm7NVRyJOTVov_5Gbl5peGe6oz5fk,12357
pymysql/constants/FIELD_TYPE.py,sha256=ytFzgAnGmb9hvdsBlnK68qdZv_a6jYFIXT6VSAb60z8,370
pymysql/constants/FLAG.py,sha256=Fy-PrCLnUI7fx_o5WypYnUAzWAM0E9d5yL8fFRVKffY,214
pymysql/constants/SERVER_STATUS.py,sha256=m28Iq5JGCFCWLhafE73-iOvw_9gDGqnytW3NkHpbugA,333
pymysql/constants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
pymysql/constants/__pycache__/CLIENT.cpython-312.pyc,,
pymysql/constants/__pycache__/COMMAND.cpython-312.pyc,,
pymysql/constants/__pycache__/CR.cpython-312.pyc,,
pymysql/constants/__pycache__/ER.cpython-312.pyc,,
pymysql/constants/__pycache__/FIELD_TYPE.cpython-312.pyc,,
pymysql/constants/__pycache__/FLAG.cpython-312.pyc,,
pymysql/constants/__pycache__/SERVER_STATUS.cpython-312.pyc,,
pymysql/constants/__pycache__/__init__.cpython-312.pyc,,
pymysql/converters.py,sha256=83EH1qFKLFSlkDAQyXbHUDUmVawmvc6jQIqq-qwQTew,9630
pymysql/cursors.py,sha256=uzArDm9ZgT7L3fA85srCBdvhqSzQ05TpNo8tJIUIzfc,16045
pymysql/err.py,sha256=bRvcfsehldXsn8dGsw4NdPlZAm8DFzRZA4qGSW4OjrU,4230
pymysql/optionfile.py,sha256=eQoz6c43yvmHtp5MI9TB2GPRdoggOLemcUWABksfutk,651
pymysql/protocol.py,sha256=aD-PGPRYcwkSI6ZJoJWZVRKn9H_A0f70KfPDu65tq0o,11812
pymysql/times.py,sha256=_qXgDaYwsHntvpIKSKXp1rrYIgtq6Z9pLyLnO2XNoL0,360
@@ -0,0 +1,5 @@
Wheel-Version: 1.0
Generator: setuptools (82.0.1)
Root-Is-Purelib: true
Tag: py3-none-any
@@ -0,0 +1,19 @@
Copyright (c) 2010, 2013 PyMySQL contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
@@ -0,0 +1 @@
pymysql
@@ -0,0 +1,183 @@
"""
PyMySQL: A pure-Python MySQL client library.
Copyright (c) 2010-2016 PyMySQL contributors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""
import sys
from .constants import FIELD_TYPE
from .err import (
Warning,
Error,
InterfaceError,
DataError,
DatabaseError,
OperationalError,
IntegrityError,
InternalError,
NotSupportedError,
ProgrammingError,
MySQLError,
)
from .times import (
Date,
Time,
Timestamp,
DateFromTicks,
TimeFromTicks,
TimestampFromTicks,
)
# PyMySQL version.
# Used by setuptools and connection_attrs
VERSION = (1, 2, 0, "final")
VERSION_STRING = "1.2.0"
### for mysqlclient compatibility
### Django checks mysqlclient version.
version_info = (2, 2, 8, "final", 1)
__version__ = "2.2.8"
def get_client_info(): # for MySQLdb compatibility
return __version__
def install_as_MySQLdb():
"""
After this function is called, any application that imports MySQLdb
will unwittingly actually use pymysql.
"""
sys.modules["MySQLdb"] = sys.modules["pymysql"]
# end of mysqlclient compatibility code
threadsafety = 1
apilevel = "2.0"
paramstyle = "pyformat"
from . import connections # noqa: E402
class DBAPISet(frozenset):
def __ne__(self, other):
if isinstance(other, set):
return frozenset.__ne__(self, other)
else:
return other not in self
def __eq__(self, other):
if isinstance(other, frozenset):
return frozenset.__eq__(self, other)
else:
return other in self
def __hash__(self):
return frozenset.__hash__(self)
STRING = DBAPISet([FIELD_TYPE.ENUM, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING])
BINARY = DBAPISet(
[
FIELD_TYPE.BLOB,
FIELD_TYPE.LONG_BLOB,
FIELD_TYPE.MEDIUM_BLOB,
FIELD_TYPE.TINY_BLOB,
]
)
NUMBER = DBAPISet(
[
FIELD_TYPE.DECIMAL,
FIELD_TYPE.DOUBLE,
FIELD_TYPE.FLOAT,
FIELD_TYPE.INT24,
FIELD_TYPE.LONG,
FIELD_TYPE.LONGLONG,
FIELD_TYPE.TINY,
FIELD_TYPE.YEAR,
]
)
DATE = DBAPISet([FIELD_TYPE.DATE, FIELD_TYPE.NEWDATE])
TIME = DBAPISet([FIELD_TYPE.TIME])
TIMESTAMP = DBAPISet([FIELD_TYPE.TIMESTAMP, FIELD_TYPE.DATETIME])
DATETIME = TIMESTAMP
ROWID = DBAPISet()
def Binary(x):
"""Return x as a binary type."""
return bytes(x)
def thread_safe():
return True # match MySQLdb.thread_safe()
Connect = connect = Connection = connections.Connection
NULL = "NULL"
__all__ = [
"BINARY",
"Binary",
"Connect",
"Connection",
"DATE",
"Date",
"Time",
"Timestamp",
"DateFromTicks",
"TimeFromTicks",
"TimestampFromTicks",
"DataError",
"DatabaseError",
"Error",
"FIELD_TYPE",
"IntegrityError",
"InterfaceError",
"InternalError",
"MySQLError",
"NULL",
"NUMBER",
"NotSupportedError",
"DBAPISet",
"OperationalError",
"ProgrammingError",
"ROWID",
"STRING",
"TIME",
"TIMESTAMP",
"Warning",
"apilevel",
"connect",
"connections",
"constants",
"converters",
"cursors",
"get_client_info",
"paramstyle",
"threadsafety",
"version_info",
"install_as_MySQLdb",
"__version__",
]
@@ -0,0 +1,272 @@
"""
Implements auth methods
"""
from .err import OperationalError
try:
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization, hashes
from cryptography.hazmat.primitives.asymmetric import padding
_have_cryptography = True
except ImportError:
_have_cryptography = False
from functools import partial
import hashlib
DEBUG = False
SCRAMBLE_LENGTH = 20
sha1_new = partial(hashlib.new, "sha1")
# mysql_native_password
# https://dev.mysql.com/doc/internals/en/secure-password-authentication.html#packet-Authentication::Native41
def scramble_native_password(password, message):
"""Scramble used for mysql_native_password"""
if not password:
return b""
stage1 = sha1_new(password).digest()
stage2 = sha1_new(stage1).digest()
s = sha1_new()
s.update(message[:SCRAMBLE_LENGTH])
s.update(stage2)
result = s.digest()
return _my_crypt(result, stage1)
def _my_crypt(message1, message2):
result = bytearray(message1)
for i in range(len(result)):
result[i] ^= message2[i]
return bytes(result)
# MariaDB's client_ed25519-plugin
# https://mariadb.com/kb/en/library/connection/#client_ed25519-plugin
_nacl_bindings = False
def _init_nacl():
global _nacl_bindings
try:
from nacl import bindings
_nacl_bindings = bindings
except ImportError:
raise RuntimeError(
"'pynacl' package is required for ed25519_password auth method"
)
def _scalar_clamp(s32):
ba = bytearray(s32)
ba0 = bytes(bytearray([ba[0] & 248]))
ba31 = bytes(bytearray([(ba[31] & 127) | 64]))
return ba0 + bytes(s32[1:31]) + ba31
def ed25519_password(password, scramble):
"""Sign a random scramble with elliptic curve Ed25519.
Secret and public key are derived from password.
"""
# variable names based on rfc8032 section-5.1.6
#
if not _nacl_bindings:
_init_nacl()
# h = SHA512(password)
h = hashlib.sha512(password).digest()
# s = prune(first_half(h))
s = _scalar_clamp(h[:32])
# r = SHA512(second_half(h) || M)
r = hashlib.sha512(h[32:] + scramble).digest()
# R = encoded point [r]B
r = _nacl_bindings.crypto_core_ed25519_scalar_reduce(r)
R = _nacl_bindings.crypto_scalarmult_ed25519_base_noclamp(r)
# A = encoded point [s]B
A = _nacl_bindings.crypto_scalarmult_ed25519_base_noclamp(s)
# k = SHA512(R || A || M)
k = hashlib.sha512(R + A + scramble).digest()
# S = (k * s + r) mod L
k = _nacl_bindings.crypto_core_ed25519_scalar_reduce(k)
ks = _nacl_bindings.crypto_core_ed25519_scalar_mul(k, s)
S = _nacl_bindings.crypto_core_ed25519_scalar_add(ks, r)
# signature = R || S
return R + S
# sha256_password
def _roundtrip(conn, send_data):
conn.write_packet(send_data)
pkt = conn._read_packet()
pkt.check_error()
return pkt
def _xor_password(password, salt):
# Trailing NUL character will be added in Auth Switch Request.
# See https://github.com/mysql/mysql-server/blob/7d10c82196c8e45554f27c00681474a9fb86d137/sql/auth/sha2_password.cc#L939-L945
salt = salt[:SCRAMBLE_LENGTH]
password_bytes = bytearray(password)
# salt = bytearray(salt) # for PY2 compat.
salt_len = len(salt)
for i in range(len(password_bytes)):
password_bytes[i] ^= salt[i % salt_len]
return bytes(password_bytes)
def sha2_rsa_encrypt(password, salt, public_key):
"""Encrypt password with salt and public_key.
Used for sha256_password and caching_sha2_password.
"""
if not _have_cryptography:
raise RuntimeError(
"'cryptography' package is required for sha256_password or"
+ " caching_sha2_password auth methods"
)
message = _xor_password(password + b"\0", salt)
rsa_key = serialization.load_pem_public_key(public_key, default_backend())
return rsa_key.encrypt(
message,
padding.OAEP(
mgf=padding.MGF1(algorithm=hashes.SHA1()),
algorithm=hashes.SHA1(),
label=None,
),
)
def sha256_password_auth(conn, pkt):
if conn._secure:
if DEBUG:
print("sha256: Sending plain password")
data = conn.password + b"\0"
return _roundtrip(conn, data)
if pkt.is_auth_switch_request():
conn.salt = pkt.read_all()
if conn.salt.endswith(b"\0"):
conn.salt = conn.salt[:-1]
if not conn.server_public_key and conn.password:
# Request server public key
if DEBUG:
print("sha256: Requesting server public key")
pkt = _roundtrip(conn, b"\1")
if pkt.is_extra_auth_data():
conn.server_public_key = pkt._data[1:]
if DEBUG:
print("Received public key:\n", conn.server_public_key.decode("ascii"))
if conn.password:
if not conn.server_public_key:
raise OperationalError("Couldn't receive server's public key")
data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key)
else:
data = b""
return _roundtrip(conn, data)
def scramble_caching_sha2(password, nonce):
# (bytes, bytes) -> bytes
"""Scramble algorithm used in cached_sha2_password fast path.
XOR(SHA256(password), SHA256(SHA256(SHA256(password)), nonce))
"""
if not password:
return b""
p1 = hashlib.sha256(password).digest()
p2 = hashlib.sha256(p1).digest()
p3 = hashlib.sha256(p2 + nonce).digest()
res = bytearray(p1)
for i in range(len(p3)):
res[i] ^= p3[i]
return bytes(res)
def caching_sha2_password_auth(conn, pkt):
# No password fast path
if not conn.password:
return _roundtrip(conn, b"")
if pkt.is_auth_switch_request():
# Try from fast auth
conn.salt = pkt.read_all()
if conn.salt.endswith(b"\0"): # str.removesuffix is available in 3.9
conn.salt = conn.salt[:-1]
if DEBUG:
print(f"caching sha2: Trying fast path. salt={conn.salt.hex()!r}")
scrambled = scramble_caching_sha2(conn.password, conn.salt)
pkt = _roundtrip(conn, scrambled)
# else: fast auth is tried in initial handshake
if not pkt.is_extra_auth_data():
raise OperationalError(
"caching sha2: Unknown packet for fast auth: %s" % pkt._data[:1]
)
# magic numbers:
# 2 - request public key
# 3 - fast auth succeeded
# 4 - need full auth
pkt.advance(1)
n = pkt.read_uint8()
if n == 3:
if DEBUG:
print("caching sha2: succeeded by fast path.")
pkt = conn._read_packet()
pkt.check_error() # pkt must be OK packet
return pkt
if n != 4:
raise OperationalError("caching sha2: Unknown result for fast auth: %s" % n)
if DEBUG:
print("caching sha2: Trying full auth...")
if conn._secure:
if DEBUG:
print("caching sha2: Sending plain password via secure connection")
return _roundtrip(conn, conn.password + b"\0")
if not conn.server_public_key:
pkt = _roundtrip(conn, b"\x02") # Request public key
if not pkt.is_extra_auth_data():
raise OperationalError(
"caching sha2: Unknown packet for public key: %s" % pkt._data[:1]
)
conn.server_public_key = pkt._data[1:]
if DEBUG:
print(conn.server_public_key.decode("ascii"))
data = sha2_rsa_encrypt(conn.password, conn.salt, conn.server_public_key)
pkt = _roundtrip(conn, data)
@@ -0,0 +1,217 @@
# Internal use only. Do not use directly.
MBLENGTH = {8: 1, 33: 3, 88: 2, 91: 2}
class Charset:
def __init__(self, id, name, collation, is_default=False):
self.id, self.name, self.collation = id, name, collation
self.is_default = is_default
def __repr__(self):
return (
f"Charset(id={self.id}, name={self.name!r}, collation={self.collation!r})"
)
@property
def encoding(self):
name = self.name
if name in ("utf8mb4", "utf8mb3"):
return "utf8"
if name == "latin1":
return "cp1252"
if name == "koi8r":
return "koi8_r"
if name == "koi8u":
return "koi8_u"
return name
@property
def is_binary(self):
return self.id == 63
class Charsets:
def __init__(self):
self._by_id = {}
self._by_name = {}
def add(self, c):
self._by_id[c.id] = c
if c.is_default:
self._by_name[c.name] = c
def by_id(self, id):
return self._by_id[id]
def by_name(self, name):
name = name.lower()
if name == "utf8":
name = "utf8mb4"
return self._by_name.get(name)
_charsets = Charsets()
charset_by_name = _charsets.by_name
charset_by_id = _charsets.by_id
"""
TODO: update this script.
Generated with:
mysql -N -s -e "select id, character_set_name, collation_name, is_default
from information_schema.collations order by id;" | python -c "import sys
for l in sys.stdin.readlines():
id, name, collation, is_default = l.split(chr(9))
if is_default.strip() == "Yes":
print('_charsets.add(Charset(%s, \'%s\', \'%s\', True))' \
% (id, name, collation))
else:
print('_charsets.add(Charset(%s, \'%s\', \'%s\'))' \
% (id, name, collation, bool(is_default.strip()))
"""
_charsets.add(Charset(1, "big5", "big5_chinese_ci", True))
_charsets.add(Charset(2, "latin2", "latin2_czech_cs"))
_charsets.add(Charset(3, "dec8", "dec8_swedish_ci", True))
_charsets.add(Charset(4, "cp850", "cp850_general_ci", True))
_charsets.add(Charset(5, "latin1", "latin1_german1_ci"))
_charsets.add(Charset(6, "hp8", "hp8_english_ci", True))
_charsets.add(Charset(7, "koi8r", "koi8r_general_ci", True))
_charsets.add(Charset(8, "latin1", "latin1_swedish_ci", True))
_charsets.add(Charset(9, "latin2", "latin2_general_ci", True))
_charsets.add(Charset(10, "swe7", "swe7_swedish_ci", True))
_charsets.add(Charset(11, "ascii", "ascii_general_ci", True))
_charsets.add(Charset(12, "ujis", "ujis_japanese_ci", True))
_charsets.add(Charset(13, "sjis", "sjis_japanese_ci", True))
_charsets.add(Charset(14, "cp1251", "cp1251_bulgarian_ci"))
_charsets.add(Charset(15, "latin1", "latin1_danish_ci"))
_charsets.add(Charset(16, "hebrew", "hebrew_general_ci", True))
_charsets.add(Charset(18, "tis620", "tis620_thai_ci", True))
_charsets.add(Charset(19, "euckr", "euckr_korean_ci", True))
_charsets.add(Charset(20, "latin7", "latin7_estonian_cs"))
_charsets.add(Charset(21, "latin2", "latin2_hungarian_ci"))
_charsets.add(Charset(22, "koi8u", "koi8u_general_ci", True))
_charsets.add(Charset(23, "cp1251", "cp1251_ukrainian_ci"))
_charsets.add(Charset(24, "gb2312", "gb2312_chinese_ci", True))
_charsets.add(Charset(25, "greek", "greek_general_ci", True))
_charsets.add(Charset(26, "cp1250", "cp1250_general_ci", True))
_charsets.add(Charset(27, "latin2", "latin2_croatian_ci"))
_charsets.add(Charset(28, "gbk", "gbk_chinese_ci", True))
_charsets.add(Charset(29, "cp1257", "cp1257_lithuanian_ci"))
_charsets.add(Charset(30, "latin5", "latin5_turkish_ci", True))
_charsets.add(Charset(31, "latin1", "latin1_german2_ci"))
_charsets.add(Charset(32, "armscii8", "armscii8_general_ci", True))
_charsets.add(Charset(33, "utf8mb3", "utf8mb3_general_ci", True))
_charsets.add(Charset(34, "cp1250", "cp1250_czech_cs"))
_charsets.add(Charset(36, "cp866", "cp866_general_ci", True))
_charsets.add(Charset(37, "keybcs2", "keybcs2_general_ci", True))
_charsets.add(Charset(38, "macce", "macce_general_ci", True))
_charsets.add(Charset(39, "macroman", "macroman_general_ci", True))
_charsets.add(Charset(40, "cp852", "cp852_general_ci", True))
_charsets.add(Charset(41, "latin7", "latin7_general_ci", True))
_charsets.add(Charset(42, "latin7", "latin7_general_cs"))
_charsets.add(Charset(43, "macce", "macce_bin"))
_charsets.add(Charset(44, "cp1250", "cp1250_croatian_ci"))
_charsets.add(Charset(45, "utf8mb4", "utf8mb4_general_ci", True))
_charsets.add(Charset(46, "utf8mb4", "utf8mb4_bin"))
_charsets.add(Charset(47, "latin1", "latin1_bin"))
_charsets.add(Charset(48, "latin1", "latin1_general_ci"))
_charsets.add(Charset(49, "latin1", "latin1_general_cs"))
_charsets.add(Charset(50, "cp1251", "cp1251_bin"))
_charsets.add(Charset(51, "cp1251", "cp1251_general_ci", True))
_charsets.add(Charset(52, "cp1251", "cp1251_general_cs"))
_charsets.add(Charset(53, "macroman", "macroman_bin"))
_charsets.add(Charset(57, "cp1256", "cp1256_general_ci", True))
_charsets.add(Charset(58, "cp1257", "cp1257_bin"))
_charsets.add(Charset(59, "cp1257", "cp1257_general_ci", True))
_charsets.add(Charset(63, "binary", "binary", True))
_charsets.add(Charset(64, "armscii8", "armscii8_bin"))
_charsets.add(Charset(65, "ascii", "ascii_bin"))
_charsets.add(Charset(66, "cp1250", "cp1250_bin"))
_charsets.add(Charset(67, "cp1256", "cp1256_bin"))
_charsets.add(Charset(68, "cp866", "cp866_bin"))
_charsets.add(Charset(69, "dec8", "dec8_bin"))
_charsets.add(Charset(70, "greek", "greek_bin"))
_charsets.add(Charset(71, "hebrew", "hebrew_bin"))
_charsets.add(Charset(72, "hp8", "hp8_bin"))
_charsets.add(Charset(73, "keybcs2", "keybcs2_bin"))
_charsets.add(Charset(74, "koi8r", "koi8r_bin"))
_charsets.add(Charset(75, "koi8u", "koi8u_bin"))
_charsets.add(Charset(76, "utf8mb3", "utf8mb3_tolower_ci"))
_charsets.add(Charset(77, "latin2", "latin2_bin"))
_charsets.add(Charset(78, "latin5", "latin5_bin"))
_charsets.add(Charset(79, "latin7", "latin7_bin"))
_charsets.add(Charset(80, "cp850", "cp850_bin"))
_charsets.add(Charset(81, "cp852", "cp852_bin"))
_charsets.add(Charset(82, "swe7", "swe7_bin"))
_charsets.add(Charset(83, "utf8mb3", "utf8mb3_bin"))
_charsets.add(Charset(84, "big5", "big5_bin"))
_charsets.add(Charset(85, "euckr", "euckr_bin"))
_charsets.add(Charset(86, "gb2312", "gb2312_bin"))
_charsets.add(Charset(87, "gbk", "gbk_bin"))
_charsets.add(Charset(88, "sjis", "sjis_bin"))
_charsets.add(Charset(89, "tis620", "tis620_bin"))
_charsets.add(Charset(91, "ujis", "ujis_bin"))
_charsets.add(Charset(92, "geostd8", "geostd8_general_ci", True))
_charsets.add(Charset(93, "geostd8", "geostd8_bin"))
_charsets.add(Charset(94, "latin1", "latin1_spanish_ci"))
_charsets.add(Charset(95, "cp932", "cp932_japanese_ci", True))
_charsets.add(Charset(96, "cp932", "cp932_bin"))
_charsets.add(Charset(97, "eucjpms", "eucjpms_japanese_ci", True))
_charsets.add(Charset(98, "eucjpms", "eucjpms_bin"))
_charsets.add(Charset(99, "cp1250", "cp1250_polish_ci"))
_charsets.add(Charset(192, "utf8mb3", "utf8mb3_unicode_ci"))
_charsets.add(Charset(193, "utf8mb3", "utf8mb3_icelandic_ci"))
_charsets.add(Charset(194, "utf8mb3", "utf8mb3_latvian_ci"))
_charsets.add(Charset(195, "utf8mb3", "utf8mb3_romanian_ci"))
_charsets.add(Charset(196, "utf8mb3", "utf8mb3_slovenian_ci"))
_charsets.add(Charset(197, "utf8mb3", "utf8mb3_polish_ci"))
_charsets.add(Charset(198, "utf8mb3", "utf8mb3_estonian_ci"))
_charsets.add(Charset(199, "utf8mb3", "utf8mb3_spanish_ci"))
_charsets.add(Charset(200, "utf8mb3", "utf8mb3_swedish_ci"))
_charsets.add(Charset(201, "utf8mb3", "utf8mb3_turkish_ci"))
_charsets.add(Charset(202, "utf8mb3", "utf8mb3_czech_ci"))
_charsets.add(Charset(203, "utf8mb3", "utf8mb3_danish_ci"))
_charsets.add(Charset(204, "utf8mb3", "utf8mb3_lithuanian_ci"))
_charsets.add(Charset(205, "utf8mb3", "utf8mb3_slovak_ci"))
_charsets.add(Charset(206, "utf8mb3", "utf8mb3_spanish2_ci"))
_charsets.add(Charset(207, "utf8mb3", "utf8mb3_roman_ci"))
_charsets.add(Charset(208, "utf8mb3", "utf8mb3_persian_ci"))
_charsets.add(Charset(209, "utf8mb3", "utf8mb3_esperanto_ci"))
_charsets.add(Charset(210, "utf8mb3", "utf8mb3_hungarian_ci"))
_charsets.add(Charset(211, "utf8mb3", "utf8mb3_sinhala_ci"))
_charsets.add(Charset(212, "utf8mb3", "utf8mb3_german2_ci"))
_charsets.add(Charset(213, "utf8mb3", "utf8mb3_croatian_ci"))
_charsets.add(Charset(214, "utf8mb3", "utf8mb3_unicode_520_ci"))
_charsets.add(Charset(215, "utf8mb3", "utf8mb3_vietnamese_ci"))
_charsets.add(Charset(223, "utf8mb3", "utf8mb3_general_mysql500_ci"))
_charsets.add(Charset(224, "utf8mb4", "utf8mb4_unicode_ci"))
_charsets.add(Charset(225, "utf8mb4", "utf8mb4_icelandic_ci"))
_charsets.add(Charset(226, "utf8mb4", "utf8mb4_latvian_ci"))
_charsets.add(Charset(227, "utf8mb4", "utf8mb4_romanian_ci"))
_charsets.add(Charset(228, "utf8mb4", "utf8mb4_slovenian_ci"))
_charsets.add(Charset(229, "utf8mb4", "utf8mb4_polish_ci"))
_charsets.add(Charset(230, "utf8mb4", "utf8mb4_estonian_ci"))
_charsets.add(Charset(231, "utf8mb4", "utf8mb4_spanish_ci"))
_charsets.add(Charset(232, "utf8mb4", "utf8mb4_swedish_ci"))
_charsets.add(Charset(233, "utf8mb4", "utf8mb4_turkish_ci"))
_charsets.add(Charset(234, "utf8mb4", "utf8mb4_czech_ci"))
_charsets.add(Charset(235, "utf8mb4", "utf8mb4_danish_ci"))
_charsets.add(Charset(236, "utf8mb4", "utf8mb4_lithuanian_ci"))
_charsets.add(Charset(237, "utf8mb4", "utf8mb4_slovak_ci"))
_charsets.add(Charset(238, "utf8mb4", "utf8mb4_spanish2_ci"))
_charsets.add(Charset(239, "utf8mb4", "utf8mb4_roman_ci"))
_charsets.add(Charset(240, "utf8mb4", "utf8mb4_persian_ci"))
_charsets.add(Charset(241, "utf8mb4", "utf8mb4_esperanto_ci"))
_charsets.add(Charset(242, "utf8mb4", "utf8mb4_hungarian_ci"))
_charsets.add(Charset(243, "utf8mb4", "utf8mb4_sinhala_ci"))
_charsets.add(Charset(244, "utf8mb4", "utf8mb4_german2_ci"))
_charsets.add(Charset(245, "utf8mb4", "utf8mb4_croatian_ci"))
_charsets.add(Charset(246, "utf8mb4", "utf8mb4_unicode_520_ci"))
_charsets.add(Charset(247, "utf8mb4", "utf8mb4_vietnamese_ci"))
_charsets.add(Charset(248, "gb18030", "gb18030_chinese_ci", True))
_charsets.add(Charset(249, "gb18030", "gb18030_bin"))
_charsets.add(Charset(250, "gb18030", "gb18030_unicode_520_ci"))
_charsets.add(Charset(255, "utf8mb4", "utf8mb4_0900_ai_ci"))
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,38 @@
# https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags
LONG_PASSWORD = 1
FOUND_ROWS = 1 << 1
LONG_FLAG = 1 << 2
CONNECT_WITH_DB = 1 << 3
NO_SCHEMA = 1 << 4
COMPRESS = 1 << 5
ODBC = 1 << 6
LOCAL_FILES = 1 << 7
IGNORE_SPACE = 1 << 8
PROTOCOL_41 = 1 << 9
INTERACTIVE = 1 << 10
SSL = 1 << 11
IGNORE_SIGPIPE = 1 << 12
TRANSACTIONS = 1 << 13
SECURE_CONNECTION = 1 << 15
MULTI_STATEMENTS = 1 << 16
MULTI_RESULTS = 1 << 17
PS_MULTI_RESULTS = 1 << 18
PLUGIN_AUTH = 1 << 19
CONNECT_ATTRS = 1 << 20
PLUGIN_AUTH_LENENC_CLIENT_DATA = 1 << 21
CAPABILITIES = (
LONG_PASSWORD
| LONG_FLAG
| PROTOCOL_41
| TRANSACTIONS
| SECURE_CONNECTION
| MULTI_RESULTS
| PLUGIN_AUTH
| PLUGIN_AUTH_LENENC_CLIENT_DATA
| CONNECT_ATTRS
)
# Not done yet
HANDLE_EXPIRED_PASSWORDS = 1 << 22
SESSION_TRACK = 1 << 23
DEPRECATE_EOF = 1 << 24
@@ -0,0 +1,32 @@
COM_SLEEP = 0x00
COM_QUIT = 0x01
COM_INIT_DB = 0x02
COM_QUERY = 0x03
COM_FIELD_LIST = 0x04
COM_CREATE_DB = 0x05
COM_DROP_DB = 0x06
COM_REFRESH = 0x07
COM_SHUTDOWN = 0x08
COM_STATISTICS = 0x09
COM_PROCESS_INFO = 0x0A
COM_CONNECT = 0x0B
COM_PROCESS_KILL = 0x0C
COM_DEBUG = 0x0D
COM_PING = 0x0E
COM_TIME = 0x0F
COM_DELAYED_INSERT = 0x10
COM_CHANGE_USER = 0x11
COM_BINLOG_DUMP = 0x12
COM_TABLE_DUMP = 0x13
COM_CONNECT_OUT = 0x14
COM_REGISTER_SLAVE = 0x15
COM_STMT_PREPARE = 0x16
COM_STMT_EXECUTE = 0x17
COM_STMT_SEND_LONG_DATA = 0x18
COM_STMT_CLOSE = 0x19
COM_STMT_RESET = 0x1A
COM_SET_OPTION = 0x1B
COM_STMT_FETCH = 0x1C
COM_DAEMON = 0x1D
COM_BINLOG_DUMP_GTID = 0x1E
COM_END = 0x1F
@@ -0,0 +1,79 @@
# flake8: noqa
# errmsg.h
CR_ERROR_FIRST = 2000
CR_UNKNOWN_ERROR = 2000
CR_SOCKET_CREATE_ERROR = 2001
CR_CONNECTION_ERROR = 2002
CR_CONN_HOST_ERROR = 2003
CR_IPSOCK_ERROR = 2004
CR_UNKNOWN_HOST = 2005
CR_SERVER_GONE_ERROR = 2006
CR_VERSION_ERROR = 2007
CR_OUT_OF_MEMORY = 2008
CR_WRONG_HOST_INFO = 2009
CR_LOCALHOST_CONNECTION = 2010
CR_TCP_CONNECTION = 2011
CR_SERVER_HANDSHAKE_ERR = 2012
CR_SERVER_LOST = 2013
CR_COMMANDS_OUT_OF_SYNC = 2014
CR_NAMEDPIPE_CONNECTION = 2015
CR_NAMEDPIPEWAIT_ERROR = 2016
CR_NAMEDPIPEOPEN_ERROR = 2017
CR_NAMEDPIPESETSTATE_ERROR = 2018
CR_CANT_READ_CHARSET = 2019
CR_NET_PACKET_TOO_LARGE = 2020
CR_EMBEDDED_CONNECTION = 2021
CR_PROBE_SLAVE_STATUS = 2022
CR_PROBE_SLAVE_HOSTS = 2023
CR_PROBE_SLAVE_CONNECT = 2024
CR_PROBE_MASTER_CONNECT = 2025
CR_SSL_CONNECTION_ERROR = 2026
CR_MALFORMED_PACKET = 2027
CR_WRONG_LICENSE = 2028
CR_NULL_POINTER = 2029
CR_NO_PREPARE_STMT = 2030
CR_PARAMS_NOT_BOUND = 2031
CR_DATA_TRUNCATED = 2032
CR_NO_PARAMETERS_EXISTS = 2033
CR_INVALID_PARAMETER_NO = 2034
CR_INVALID_BUFFER_USE = 2035
CR_UNSUPPORTED_PARAM_TYPE = 2036
CR_SHARED_MEMORY_CONNECTION = 2037
CR_SHARED_MEMORY_CONNECT_REQUEST_ERROR = 2038
CR_SHARED_MEMORY_CONNECT_ANSWER_ERROR = 2039
CR_SHARED_MEMORY_CONNECT_FILE_MAP_ERROR = 2040
CR_SHARED_MEMORY_CONNECT_MAP_ERROR = 2041
CR_SHARED_MEMORY_FILE_MAP_ERROR = 2042
CR_SHARED_MEMORY_MAP_ERROR = 2043
CR_SHARED_MEMORY_EVENT_ERROR = 2044
CR_SHARED_MEMORY_CONNECT_ABANDONED_ERROR = 2045
CR_SHARED_MEMORY_CONNECT_SET_ERROR = 2046
CR_CONN_UNKNOW_PROTOCOL = 2047
CR_INVALID_CONN_HANDLE = 2048
CR_SECURE_AUTH = 2049
CR_FETCH_CANCELED = 2050
CR_NO_DATA = 2051
CR_NO_STMT_METADATA = 2052
CR_NO_RESULT_SET = 2053
CR_NOT_IMPLEMENTED = 2054
CR_SERVER_LOST_EXTENDED = 2055
CR_STMT_CLOSED = 2056
CR_NEW_STMT_METADATA = 2057
CR_ALREADY_CONNECTED = 2058
CR_AUTH_PLUGIN_CANNOT_LOAD = 2059
CR_DUPLICATE_CONNECTION_ATTR = 2060
CR_AUTH_PLUGIN_ERR = 2061
CR_INSECURE_API_ERR = 2062
CR_FILE_NAME_TOO_LONG = 2063
CR_SSL_FIPS_MODE_ERR = 2064
CR_DEPRECATED_COMPRESSION_NOT_SUPPORTED = 2065
CR_COMPRESSION_WRONGLY_CONFIGURED = 2066
CR_KERBEROS_USER_NOT_FOUND = 2067
CR_LOAD_DATA_LOCAL_INFILE_REJECTED = 2068
CR_LOAD_DATA_LOCAL_INFILE_REALPATH_FAIL = 2069
CR_DNS_SRV_LOOKUP_FAILED = 2070
CR_MANDATORY_TRACKER_NOT_FOUND = 2071
CR_INVALID_FACTOR_NO = 2072
CR_ERROR_LAST = 2072
@@ -0,0 +1,477 @@
ERROR_FIRST = 1000
HASHCHK = 1000
NISAMCHK = 1001
NO = 1002
YES = 1003
CANT_CREATE_FILE = 1004
CANT_CREATE_TABLE = 1005
CANT_CREATE_DB = 1006
DB_CREATE_EXISTS = 1007
DB_DROP_EXISTS = 1008
DB_DROP_DELETE = 1009
DB_DROP_RMDIR = 1010
CANT_DELETE_FILE = 1011
CANT_FIND_SYSTEM_REC = 1012
CANT_GET_STAT = 1013
CANT_GET_WD = 1014
CANT_LOCK = 1015
CANT_OPEN_FILE = 1016
FILE_NOT_FOUND = 1017
CANT_READ_DIR = 1018
CANT_SET_WD = 1019
CHECKREAD = 1020
DISK_FULL = 1021
DUP_KEY = 1022
ERROR_ON_CLOSE = 1023
ERROR_ON_READ = 1024
ERROR_ON_RENAME = 1025
ERROR_ON_WRITE = 1026
FILE_USED = 1027
FILSORT_ABORT = 1028
FORM_NOT_FOUND = 1029
GET_ERRNO = 1030
ILLEGAL_HA = 1031
KEY_NOT_FOUND = 1032
NOT_FORM_FILE = 1033
NOT_KEYFILE = 1034
OLD_KEYFILE = 1035
OPEN_AS_READONLY = 1036
OUTOFMEMORY = 1037
OUT_OF_SORTMEMORY = 1038
UNEXPECTED_EOF = 1039
CON_COUNT_ERROR = 1040
OUT_OF_RESOURCES = 1041
BAD_HOST_ERROR = 1042
HANDSHAKE_ERROR = 1043
DBACCESS_DENIED_ERROR = 1044
ACCESS_DENIED_ERROR = 1045
NO_DB_ERROR = 1046
UNKNOWN_COM_ERROR = 1047
BAD_NULL_ERROR = 1048
BAD_DB_ERROR = 1049
TABLE_EXISTS_ERROR = 1050
BAD_TABLE_ERROR = 1051
NON_UNIQ_ERROR = 1052
SERVER_SHUTDOWN = 1053
BAD_FIELD_ERROR = 1054
WRONG_FIELD_WITH_GROUP = 1055
WRONG_GROUP_FIELD = 1056
WRONG_SUM_SELECT = 1057
WRONG_VALUE_COUNT = 1058
TOO_LONG_IDENT = 1059
DUP_FIELDNAME = 1060
DUP_KEYNAME = 1061
DUP_ENTRY = 1062
WRONG_FIELD_SPEC = 1063
PARSE_ERROR = 1064
EMPTY_QUERY = 1065
NONUNIQ_TABLE = 1066
INVALID_DEFAULT = 1067
MULTIPLE_PRI_KEY = 1068
TOO_MANY_KEYS = 1069
TOO_MANY_KEY_PARTS = 1070
TOO_LONG_KEY = 1071
KEY_COLUMN_DOES_NOT_EXITS = 1072
BLOB_USED_AS_KEY = 1073
TOO_BIG_FIELDLENGTH = 1074
WRONG_AUTO_KEY = 1075
READY = 1076
NORMAL_SHUTDOWN = 1077
GOT_SIGNAL = 1078
SHUTDOWN_COMPLETE = 1079
FORCING_CLOSE = 1080
IPSOCK_ERROR = 1081
NO_SUCH_INDEX = 1082
WRONG_FIELD_TERMINATORS = 1083
BLOBS_AND_NO_TERMINATED = 1084
TEXTFILE_NOT_READABLE = 1085
FILE_EXISTS_ERROR = 1086
LOAD_INFO = 1087
ALTER_INFO = 1088
WRONG_SUB_KEY = 1089
CANT_REMOVE_ALL_FIELDS = 1090
CANT_DROP_FIELD_OR_KEY = 1091
INSERT_INFO = 1092
UPDATE_TABLE_USED = 1093
NO_SUCH_THREAD = 1094
KILL_DENIED_ERROR = 1095
NO_TABLES_USED = 1096
TOO_BIG_SET = 1097
NO_UNIQUE_LOGFILE = 1098
TABLE_NOT_LOCKED_FOR_WRITE = 1099
TABLE_NOT_LOCKED = 1100
BLOB_CANT_HAVE_DEFAULT = 1101
WRONG_DB_NAME = 1102
WRONG_TABLE_NAME = 1103
TOO_BIG_SELECT = 1104
UNKNOWN_ERROR = 1105
UNKNOWN_PROCEDURE = 1106
WRONG_PARAMCOUNT_TO_PROCEDURE = 1107
WRONG_PARAMETERS_TO_PROCEDURE = 1108
UNKNOWN_TABLE = 1109
FIELD_SPECIFIED_TWICE = 1110
INVALID_GROUP_FUNC_USE = 1111
UNSUPPORTED_EXTENSION = 1112
TABLE_MUST_HAVE_COLUMNS = 1113
RECORD_FILE_FULL = 1114
UNKNOWN_CHARACTER_SET = 1115
TOO_MANY_TABLES = 1116
TOO_MANY_FIELDS = 1117
TOO_BIG_ROWSIZE = 1118
STACK_OVERRUN = 1119
WRONG_OUTER_JOIN = 1120
NULL_COLUMN_IN_INDEX = 1121
CANT_FIND_UDF = 1122
CANT_INITIALIZE_UDF = 1123
UDF_NO_PATHS = 1124
UDF_EXISTS = 1125
CANT_OPEN_LIBRARY = 1126
CANT_FIND_DL_ENTRY = 1127
FUNCTION_NOT_DEFINED = 1128
HOST_IS_BLOCKED = 1129
HOST_NOT_PRIVILEGED = 1130
PASSWORD_ANONYMOUS_USER = 1131
PASSWORD_NOT_ALLOWED = 1132
PASSWORD_NO_MATCH = 1133
UPDATE_INFO = 1134
CANT_CREATE_THREAD = 1135
WRONG_VALUE_COUNT_ON_ROW = 1136
CANT_REOPEN_TABLE = 1137
INVALID_USE_OF_NULL = 1138
REGEXP_ERROR = 1139
MIX_OF_GROUP_FUNC_AND_FIELDS = 1140
NONEXISTING_GRANT = 1141
TABLEACCESS_DENIED_ERROR = 1142
COLUMNACCESS_DENIED_ERROR = 1143
ILLEGAL_GRANT_FOR_TABLE = 1144
GRANT_WRONG_HOST_OR_USER = 1145
NO_SUCH_TABLE = 1146
NONEXISTING_TABLE_GRANT = 1147
NOT_ALLOWED_COMMAND = 1148
SYNTAX_ERROR = 1149
DELAYED_CANT_CHANGE_LOCK = 1150
TOO_MANY_DELAYED_THREADS = 1151
ABORTING_CONNECTION = 1152
NET_PACKET_TOO_LARGE = 1153
NET_READ_ERROR_FROM_PIPE = 1154
NET_FCNTL_ERROR = 1155
NET_PACKETS_OUT_OF_ORDER = 1156
NET_UNCOMPRESS_ERROR = 1157
NET_READ_ERROR = 1158
NET_READ_INTERRUPTED = 1159
NET_ERROR_ON_WRITE = 1160
NET_WRITE_INTERRUPTED = 1161
TOO_LONG_STRING = 1162
TABLE_CANT_HANDLE_BLOB = 1163
TABLE_CANT_HANDLE_AUTO_INCREMENT = 1164
DELAYED_INSERT_TABLE_LOCKED = 1165
WRONG_COLUMN_NAME = 1166
WRONG_KEY_COLUMN = 1167
WRONG_MRG_TABLE = 1168
DUP_UNIQUE = 1169
BLOB_KEY_WITHOUT_LENGTH = 1170
PRIMARY_CANT_HAVE_NULL = 1171
TOO_MANY_ROWS = 1172
REQUIRES_PRIMARY_KEY = 1173
NO_RAID_COMPILED = 1174
UPDATE_WITHOUT_KEY_IN_SAFE_MODE = 1175
KEY_DOES_NOT_EXITS = 1176
CHECK_NO_SUCH_TABLE = 1177
CHECK_NOT_IMPLEMENTED = 1178
CANT_DO_THIS_DURING_AN_TRANSACTION = 1179
ERROR_DURING_COMMIT = 1180
ERROR_DURING_ROLLBACK = 1181
ERROR_DURING_FLUSH_LOGS = 1182
ERROR_DURING_CHECKPOINT = 1183
NEW_ABORTING_CONNECTION = 1184
DUMP_NOT_IMPLEMENTED = 1185
FLUSH_MASTER_BINLOG_CLOSED = 1186
INDEX_REBUILD = 1187
MASTER = 1188
MASTER_NET_READ = 1189
MASTER_NET_WRITE = 1190
FT_MATCHING_KEY_NOT_FOUND = 1191
LOCK_OR_ACTIVE_TRANSACTION = 1192
UNKNOWN_SYSTEM_VARIABLE = 1193
CRASHED_ON_USAGE = 1194
CRASHED_ON_REPAIR = 1195
WARNING_NOT_COMPLETE_ROLLBACK = 1196
TRANS_CACHE_FULL = 1197
SLAVE_MUST_STOP = 1198
SLAVE_NOT_RUNNING = 1199
BAD_SLAVE = 1200
MASTER_INFO = 1201
SLAVE_THREAD = 1202
TOO_MANY_USER_CONNECTIONS = 1203
SET_CONSTANTS_ONLY = 1204
LOCK_WAIT_TIMEOUT = 1205
LOCK_TABLE_FULL = 1206
READ_ONLY_TRANSACTION = 1207
DROP_DB_WITH_READ_LOCK = 1208
CREATE_DB_WITH_READ_LOCK = 1209
WRONG_ARGUMENTS = 1210
NO_PERMISSION_TO_CREATE_USER = 1211
UNION_TABLES_IN_DIFFERENT_DIR = 1212
LOCK_DEADLOCK = 1213
TABLE_CANT_HANDLE_FT = 1214
CANNOT_ADD_FOREIGN = 1215
NO_REFERENCED_ROW = 1216
ROW_IS_REFERENCED = 1217
CONNECT_TO_MASTER = 1218
QUERY_ON_MASTER = 1219
ERROR_WHEN_EXECUTING_COMMAND = 1220
WRONG_USAGE = 1221
WRONG_NUMBER_OF_COLUMNS_IN_SELECT = 1222
CANT_UPDATE_WITH_READLOCK = 1223
MIXING_NOT_ALLOWED = 1224
DUP_ARGUMENT = 1225
USER_LIMIT_REACHED = 1226
SPECIFIC_ACCESS_DENIED_ERROR = 1227
LOCAL_VARIABLE = 1228
GLOBAL_VARIABLE = 1229
NO_DEFAULT = 1230
WRONG_VALUE_FOR_VAR = 1231
WRONG_TYPE_FOR_VAR = 1232
VAR_CANT_BE_READ = 1233
CANT_USE_OPTION_HERE = 1234
NOT_SUPPORTED_YET = 1235
MASTER_FATAL_ERROR_READING_BINLOG = 1236
SLAVE_IGNORED_TABLE = 1237
INCORRECT_GLOBAL_LOCAL_VAR = 1238
WRONG_FK_DEF = 1239
KEY_REF_DO_NOT_MATCH_TABLE_REF = 1240
OPERAND_COLUMNS = 1241
SUBQUERY_NO_1_ROW = 1242
UNKNOWN_STMT_HANDLER = 1243
CORRUPT_HELP_DB = 1244
CYCLIC_REFERENCE = 1245
AUTO_CONVERT = 1246
ILLEGAL_REFERENCE = 1247
DERIVED_MUST_HAVE_ALIAS = 1248
SELECT_REDUCED = 1249
TABLENAME_NOT_ALLOWED_HERE = 1250
NOT_SUPPORTED_AUTH_MODE = 1251
SPATIAL_CANT_HAVE_NULL = 1252
COLLATION_CHARSET_MISMATCH = 1253
SLAVE_WAS_RUNNING = 1254
SLAVE_WAS_NOT_RUNNING = 1255
TOO_BIG_FOR_UNCOMPRESS = 1256
ZLIB_Z_MEM_ERROR = 1257
ZLIB_Z_BUF_ERROR = 1258
ZLIB_Z_DATA_ERROR = 1259
CUT_VALUE_GROUP_CONCAT = 1260
WARN_TOO_FEW_RECORDS = 1261
WARN_TOO_MANY_RECORDS = 1262
WARN_NULL_TO_NOTNULL = 1263
WARN_DATA_OUT_OF_RANGE = 1264
WARN_DATA_TRUNCATED = 1265
WARN_USING_OTHER_HANDLER = 1266
CANT_AGGREGATE_2COLLATIONS = 1267
DROP_USER = 1268
REVOKE_GRANTS = 1269
CANT_AGGREGATE_3COLLATIONS = 1270
CANT_AGGREGATE_NCOLLATIONS = 1271
VARIABLE_IS_NOT_STRUCT = 1272
UNKNOWN_COLLATION = 1273
SLAVE_IGNORED_SSL_PARAMS = 1274
SERVER_IS_IN_SECURE_AUTH_MODE = 1275
WARN_FIELD_RESOLVED = 1276
BAD_SLAVE_UNTIL_COND = 1277
MISSING_SKIP_SLAVE = 1278
UNTIL_COND_IGNORED = 1279
WRONG_NAME_FOR_INDEX = 1280
WRONG_NAME_FOR_CATALOG = 1281
WARN_QC_RESIZE = 1282
BAD_FT_COLUMN = 1283
UNKNOWN_KEY_CACHE = 1284
WARN_HOSTNAME_WONT_WORK = 1285
UNKNOWN_STORAGE_ENGINE = 1286
WARN_DEPRECATED_SYNTAX = 1287
NON_UPDATABLE_TABLE = 1288
FEATURE_DISABLED = 1289
OPTION_PREVENTS_STATEMENT = 1290
DUPLICATED_VALUE_IN_TYPE = 1291
TRUNCATED_WRONG_VALUE = 1292
TOO_MUCH_AUTO_TIMESTAMP_COLS = 1293
INVALID_ON_UPDATE = 1294
UNSUPPORTED_PS = 1295
GET_ERRMSG = 1296
GET_TEMPORARY_ERRMSG = 1297
UNKNOWN_TIME_ZONE = 1298
WARN_INVALID_TIMESTAMP = 1299
INVALID_CHARACTER_STRING = 1300
WARN_ALLOWED_PACKET_OVERFLOWED = 1301
CONFLICTING_DECLARATIONS = 1302
SP_NO_RECURSIVE_CREATE = 1303
SP_ALREADY_EXISTS = 1304
SP_DOES_NOT_EXIST = 1305
SP_DROP_FAILED = 1306
SP_STORE_FAILED = 1307
SP_LILABEL_MISMATCH = 1308
SP_LABEL_REDEFINE = 1309
SP_LABEL_MISMATCH = 1310
SP_UNINIT_VAR = 1311
SP_BADSELECT = 1312
SP_BADRETURN = 1313
SP_BADSTATEMENT = 1314
UPDATE_LOG_DEPRECATED_IGNORED = 1315
UPDATE_LOG_DEPRECATED_TRANSLATED = 1316
QUERY_INTERRUPTED = 1317
SP_WRONG_NO_OF_ARGS = 1318
SP_COND_MISMATCH = 1319
SP_NORETURN = 1320
SP_NORETURNEND = 1321
SP_BAD_CURSOR_QUERY = 1322
SP_BAD_CURSOR_SELECT = 1323
SP_CURSOR_MISMATCH = 1324
SP_CURSOR_ALREADY_OPEN = 1325
SP_CURSOR_NOT_OPEN = 1326
SP_UNDECLARED_VAR = 1327
SP_WRONG_NO_OF_FETCH_ARGS = 1328
SP_FETCH_NO_DATA = 1329
SP_DUP_PARAM = 1330
SP_DUP_VAR = 1331
SP_DUP_COND = 1332
SP_DUP_CURS = 1333
SP_CANT_ALTER = 1334
SP_SUBSELECT_NYI = 1335
STMT_NOT_ALLOWED_IN_SF_OR_TRG = 1336
SP_VARCOND_AFTER_CURSHNDLR = 1337
SP_CURSOR_AFTER_HANDLER = 1338
SP_CASE_NOT_FOUND = 1339
FPARSER_TOO_BIG_FILE = 1340
FPARSER_BAD_HEADER = 1341
FPARSER_EOF_IN_COMMENT = 1342
FPARSER_ERROR_IN_PARAMETER = 1343
FPARSER_EOF_IN_UNKNOWN_PARAMETER = 1344
VIEW_NO_EXPLAIN = 1345
FRM_UNKNOWN_TYPE = 1346
WRONG_OBJECT = 1347
NONUPDATEABLE_COLUMN = 1348
VIEW_SELECT_DERIVED = 1349
VIEW_SELECT_CLAUSE = 1350
VIEW_SELECT_VARIABLE = 1351
VIEW_SELECT_TMPTABLE = 1352
VIEW_WRONG_LIST = 1353
WARN_VIEW_MERGE = 1354
WARN_VIEW_WITHOUT_KEY = 1355
VIEW_INVALID = 1356
SP_NO_DROP_SP = 1357
SP_GOTO_IN_HNDLR = 1358
TRG_ALREADY_EXISTS = 1359
TRG_DOES_NOT_EXIST = 1360
TRG_ON_VIEW_OR_TEMP_TABLE = 1361
TRG_CANT_CHANGE_ROW = 1362
TRG_NO_SUCH_ROW_IN_TRG = 1363
NO_DEFAULT_FOR_FIELD = 1364
DIVISION_BY_ZERO = 1365
TRUNCATED_WRONG_VALUE_FOR_FIELD = 1366
ILLEGAL_VALUE_FOR_TYPE = 1367
VIEW_NONUPD_CHECK = 1368
VIEW_CHECK_FAILED = 1369
PROCACCESS_DENIED_ERROR = 1370
RELAY_LOG_FAIL = 1371
PASSWD_LENGTH = 1372
UNKNOWN_TARGET_BINLOG = 1373
IO_ERR_LOG_INDEX_READ = 1374
BINLOG_PURGE_PROHIBITED = 1375
FSEEK_FAIL = 1376
BINLOG_PURGE_FATAL_ERR = 1377
LOG_IN_USE = 1378
LOG_PURGE_UNKNOWN_ERR = 1379
RELAY_LOG_INIT = 1380
NO_BINARY_LOGGING = 1381
RESERVED_SYNTAX = 1382
WSAS_FAILED = 1383
DIFF_GROUPS_PROC = 1384
NO_GROUP_FOR_PROC = 1385
ORDER_WITH_PROC = 1386
LOGGING_PROHIBIT_CHANGING_OF = 1387
NO_FILE_MAPPING = 1388
WRONG_MAGIC = 1389
PS_MANY_PARAM = 1390
KEY_PART_0 = 1391
VIEW_CHECKSUM = 1392
VIEW_MULTIUPDATE = 1393
VIEW_NO_INSERT_FIELD_LIST = 1394
VIEW_DELETE_MERGE_VIEW = 1395
CANNOT_USER = 1396
XAER_NOTA = 1397
XAER_INVAL = 1398
XAER_RMFAIL = 1399
XAER_OUTSIDE = 1400
XAER_RMERR = 1401
XA_RBROLLBACK = 1402
NONEXISTING_PROC_GRANT = 1403
PROC_AUTO_GRANT_FAIL = 1404
PROC_AUTO_REVOKE_FAIL = 1405
DATA_TOO_LONG = 1406
SP_BAD_SQLSTATE = 1407
STARTUP = 1408
LOAD_FROM_FIXED_SIZE_ROWS_TO_VAR = 1409
CANT_CREATE_USER_WITH_GRANT = 1410
WRONG_VALUE_FOR_TYPE = 1411
TABLE_DEF_CHANGED = 1412
SP_DUP_HANDLER = 1413
SP_NOT_VAR_ARG = 1414
SP_NO_RETSET = 1415
CANT_CREATE_GEOMETRY_OBJECT = 1416
FAILED_ROUTINE_BREAK_BINLOG = 1417
BINLOG_UNSAFE_ROUTINE = 1418
BINLOG_CREATE_ROUTINE_NEED_SUPER = 1419
EXEC_STMT_WITH_OPEN_CURSOR = 1420
STMT_HAS_NO_OPEN_CURSOR = 1421
COMMIT_NOT_ALLOWED_IN_SF_OR_TRG = 1422
NO_DEFAULT_FOR_VIEW_FIELD = 1423
SP_NO_RECURSION = 1424
TOO_BIG_SCALE = 1425
TOO_BIG_PRECISION = 1426
M_BIGGER_THAN_D = 1427
WRONG_LOCK_OF_SYSTEM_TABLE = 1428
CONNECT_TO_FOREIGN_DATA_SOURCE = 1429
QUERY_ON_FOREIGN_DATA_SOURCE = 1430
FOREIGN_DATA_SOURCE_DOESNT_EXIST = 1431
FOREIGN_DATA_STRING_INVALID_CANT_CREATE = 1432
FOREIGN_DATA_STRING_INVALID = 1433
CANT_CREATE_FEDERATED_TABLE = 1434
TRG_IN_WRONG_SCHEMA = 1435
STACK_OVERRUN_NEED_MORE = 1436
TOO_LONG_BODY = 1437
WARN_CANT_DROP_DEFAULT_KEYCACHE = 1438
TOO_BIG_DISPLAYWIDTH = 1439
XAER_DUPID = 1440
DATETIME_FUNCTION_OVERFLOW = 1441
CANT_UPDATE_USED_TABLE_IN_SF_OR_TRG = 1442
VIEW_PREVENT_UPDATE = 1443
PS_NO_RECURSION = 1444
SP_CANT_SET_AUTOCOMMIT = 1445
MALFORMED_DEFINER = 1446
VIEW_FRM_NO_USER = 1447
VIEW_OTHER_USER = 1448
NO_SUCH_USER = 1449
FORBID_SCHEMA_CHANGE = 1450
ROW_IS_REFERENCED_2 = 1451
NO_REFERENCED_ROW_2 = 1452
SP_BAD_VAR_SHADOW = 1453
TRG_NO_DEFINER = 1454
OLD_FILE_FORMAT = 1455
SP_RECURSION_LIMIT = 1456
SP_PROC_TABLE_CORRUPT = 1457
SP_WRONG_NAME = 1458
TABLE_NEEDS_UPGRADE = 1459
SP_NO_AGGREGATE = 1460
MAX_PREPARED_STMT_COUNT_REACHED = 1461
VIEW_RECURSIVE = 1462
NON_GROUPING_FIELD_USED = 1463
TABLE_CANT_HANDLE_SPKEYS = 1464
NO_TRIGGERS_ON_SYSTEM_SCHEMA = 1465
USERNAME = 1466
HOSTNAME = 1467
WRONG_STRING_LENGTH = 1468
ERROR_LAST = 1468
# MariaDB only
STATEMENT_TIMEOUT = 1969
QUERY_TIMEOUT = 3024
# https://github.com/PyMySQL/PyMySQL/issues/607
CONSTRAINT_FAILED = 4025
@@ -0,0 +1,31 @@
DECIMAL = 0
TINY = 1
SHORT = 2
LONG = 3
FLOAT = 4
DOUBLE = 5
NULL = 6
TIMESTAMP = 7
LONGLONG = 8
INT24 = 9
DATE = 10
TIME = 11
DATETIME = 12
YEAR = 13
NEWDATE = 14
VARCHAR = 15
BIT = 16
JSON = 245
NEWDECIMAL = 246
ENUM = 247
SET = 248
TINY_BLOB = 249
MEDIUM_BLOB = 250
LONG_BLOB = 251
BLOB = 252
VAR_STRING = 253
STRING = 254
GEOMETRY = 255
CHAR = TINY
INTERVAL = ENUM
@@ -0,0 +1,15 @@
NOT_NULL = 1
PRI_KEY = 2
UNIQUE_KEY = 4
MULTIPLE_KEY = 8
BLOB = 16
UNSIGNED = 32
ZEROFILL = 64
BINARY = 128
ENUM = 256
AUTO_INCREMENT = 512
TIMESTAMP = 1024
SET = 2048
PART_KEY = 16384
GROUP = 32767
UNIQUE = 65536
@@ -0,0 +1,10 @@
SERVER_STATUS_IN_TRANS = 1
SERVER_STATUS_AUTOCOMMIT = 2
SERVER_MORE_RESULTS_EXISTS = 8
SERVER_QUERY_NO_GOOD_INDEX_USED = 16
SERVER_QUERY_NO_INDEX_USED = 32
SERVER_STATUS_CURSOR_EXISTS = 64
SERVER_STATUS_LAST_ROW_SENT = 128
SERVER_STATUS_DB_DROPPED = 256
SERVER_STATUS_NO_BACKSLASH_ESCAPES = 512
SERVER_STATUS_METADATA_CHANGED = 1024
@@ -0,0 +1,365 @@
import datetime
from decimal import Decimal
import re
import time
from .err import ProgrammingError
from .constants import FIELD_TYPE
def escape_item(val, charset, mapping=None):
if mapping is None:
mapping = encoders
encoder = mapping.get(type(val))
# Fallback to default when no encoder found
if not encoder:
try:
encoder = mapping[str]
except KeyError:
raise TypeError("no default type converter defined")
if encoder in (escape_dict, escape_sequence):
val = encoder(val, charset, mapping)
else:
val = encoder(val, mapping)
return val
def escape_dict(val, charset, mapping=None):
raise TypeError("dict can not be used as parameter")
def escape_sequence(val, charset, mapping=None):
n = []
for item in val:
quoted = escape_item(item, charset, mapping)
n.append(quoted)
return "(" + ",".join(n) + ")"
def escape_set(val, charset, mapping=None):
return ",".join([escape_item(x, charset, mapping) for x in val])
def escape_bool(value, mapping=None):
return str(int(value))
def escape_int(value, mapping=None):
return str(value)
def escape_float(value, mapping=None):
s = repr(value)
if s in ("inf", "-inf", "nan"):
raise ProgrammingError("%s can not be used with MySQL" % s)
if "e" not in s:
s += "e0"
return s
_escape_table = [chr(x) for x in range(128)]
_escape_table[0] = "\\0"
_escape_table[ord("\\")] = "\\\\"
_escape_table[ord("\n")] = "\\n"
_escape_table[ord("\r")] = "\\r"
_escape_table[ord("\032")] = "\\Z"
_escape_table[ord('"')] = '\\"'
_escape_table[ord("'")] = "\\'"
def escape_string(value, mapping=None):
"""escapes *value* without adding quote.
Value should be unicode
"""
return value.translate(_escape_table)
def escape_bytes_prefixed(value, mapping=None):
return "_binary'%s'" % value.decode("ascii", "surrogateescape").translate(
_escape_table
)
def escape_bytes(value, mapping=None):
return "'%s'" % value.decode("ascii", "surrogateescape").translate(_escape_table)
def escape_str(value, mapping=None):
return "'%s'" % escape_string(str(value), mapping)
def escape_None(value, mapping=None):
return "NULL"
def escape_timedelta(obj, mapping=None):
seconds = int(obj.seconds) % 60
minutes = int(obj.seconds // 60) % 60
hours = int(obj.seconds // 3600) % 24 + int(obj.days) * 24
if obj.microseconds:
fmt = "'{0:02d}:{1:02d}:{2:02d}.{3:06d}'"
else:
fmt = "'{0:02d}:{1:02d}:{2:02d}'"
return fmt.format(hours, minutes, seconds, obj.microseconds)
def escape_time(obj, mapping=None):
if obj.microsecond:
fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'"
else:
fmt = "'{0.hour:02}:{0.minute:02}:{0.second:02}'"
return fmt.format(obj)
def escape_datetime(obj, mapping=None):
if obj.microsecond:
fmt = (
"'{0.year:04}-{0.month:02}-{0.day:02}"
+ " {0.hour:02}:{0.minute:02}:{0.second:02}.{0.microsecond:06}'"
)
else:
fmt = "'{0.year:04}-{0.month:02}-{0.day:02} {0.hour:02}:{0.minute:02}:{0.second:02}'"
return fmt.format(obj)
def escape_date(obj, mapping=None):
fmt = "'{0.year:04}-{0.month:02}-{0.day:02}'"
return fmt.format(obj)
def escape_struct_time(obj, mapping=None):
return escape_datetime(datetime.datetime(*obj[:6]))
def Decimal2Literal(o, d):
if not o.is_finite():
raise ProgrammingError("%s can not be used with MySQL" % str(o).lower())
return format(o, "f")
def _convert_second_fraction(s):
if not s:
return 0
# Pad zeros to ensure the fraction length in microseconds
s = s.ljust(6, "0")
return int(s[:6])
DATETIME_RE = re.compile(
r"(\d{1,4})-(\d{1,2})-(\d{1,2})[T ](\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?"
)
def convert_datetime(obj):
"""Returns a DATETIME or TIMESTAMP column value as a datetime object:
>>> convert_datetime('2007-02-25 23:06:20')
datetime.datetime(2007, 2, 25, 23, 6, 20)
>>> convert_datetime('2007-02-25T23:06:20')
datetime.datetime(2007, 2, 25, 23, 6, 20)
Illegal values are returned as str:
>>> convert_datetime('2007-02-31T23:06:20')
'2007-02-31T23:06:20'
>>> convert_datetime('0000-00-00 00:00:00')
'0000-00-00 00:00:00'
"""
if isinstance(obj, (bytes, bytearray)):
obj = obj.decode("ascii")
m = DATETIME_RE.match(obj)
if not m:
return convert_date(obj)
try:
groups = list(m.groups())
groups[-1] = _convert_second_fraction(groups[-1])
return datetime.datetime(*[int(x) for x in groups])
except ValueError:
return convert_date(obj)
TIMEDELTA_RE = re.compile(r"(-)?(\d{1,3}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
def convert_timedelta(obj):
"""Returns a TIME column as a timedelta object:
>>> convert_timedelta('25:06:17')
datetime.timedelta(days=1, seconds=3977)
>>> convert_timedelta('-25:06:17')
datetime.timedelta(days=-2, seconds=82423)
Illegal values are returned as string:
>>> convert_timedelta('random crap')
'random crap'
Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but
can accept values as (+|-)DD HH:MM:SS. The latter format will not
be parsed correctly by this function.
"""
if isinstance(obj, (bytes, bytearray)):
obj = obj.decode("ascii")
m = TIMEDELTA_RE.match(obj)
if not m:
return obj
try:
groups = list(m.groups())
groups[-1] = _convert_second_fraction(groups[-1])
negate = -1 if groups[0] else 1
hours, minutes, seconds, microseconds = groups[1:]
tdelta = (
datetime.timedelta(
hours=int(hours),
minutes=int(minutes),
seconds=int(seconds),
microseconds=int(microseconds),
)
* negate
)
return tdelta
except ValueError:
return obj
TIME_RE = re.compile(r"(\d{1,2}):(\d{1,2}):(\d{1,2})(?:.(\d{1,6}))?")
def convert_time(obj):
"""Returns a TIME column as a time object:
>>> convert_time('15:06:17')
datetime.time(15, 6, 17)
Illegal values are returned as str:
>>> convert_time('-25:06:17')
'-25:06:17'
>>> convert_time('random crap')
'random crap'
Note that MySQL always returns TIME columns as (+|-)HH:MM:SS, but
can accept values as (+|-)DD HH:MM:SS. The latter format will not
be parsed correctly by this function.
Also note that MySQL's TIME column corresponds more closely to
Python's timedelta and not time. However if you want TIME columns
to be treated as time-of-day and not a time offset, then you can
use set this function as the converter for FIELD_TYPE.TIME.
"""
if isinstance(obj, (bytes, bytearray)):
obj = obj.decode("ascii")
m = TIME_RE.match(obj)
if not m:
return obj
try:
groups = list(m.groups())
groups[-1] = _convert_second_fraction(groups[-1])
hours, minutes, seconds, microseconds = groups
return datetime.time(
hour=int(hours),
minute=int(minutes),
second=int(seconds),
microsecond=int(microseconds),
)
except ValueError:
return obj
def convert_date(obj):
"""Returns a DATE column as a date object:
>>> convert_date('2007-02-26')
datetime.date(2007, 2, 26)
Illegal values are returned as str:
>>> convert_date('2007-02-31')
'2007-02-31'
>>> convert_date('0000-00-00')
'0000-00-00'
"""
if isinstance(obj, (bytes, bytearray)):
obj = obj.decode("ascii")
try:
return datetime.date(*[int(x) for x in obj.split("-", 2)])
except ValueError:
return obj
def through(x):
return x
# def convert_bit(b):
# b = "\x00" * (8 - len(b)) + b # pad w/ zeroes
# return struct.unpack(">Q", b)[0]
#
# the snippet above is right, but MySQLdb doesn't process bits,
# so we shouldn't either
convert_bit = through
encoders = {
bool: escape_bool,
int: escape_int,
float: escape_float,
str: escape_str,
bytes: escape_bytes,
tuple: escape_sequence,
list: escape_sequence,
set: escape_sequence,
frozenset: escape_sequence,
dict: escape_dict,
type(None): escape_None,
datetime.date: escape_date,
datetime.datetime: escape_datetime,
datetime.timedelta: escape_timedelta,
datetime.time: escape_time,
time.struct_time: escape_struct_time,
Decimal: Decimal2Literal,
}
decoders = {
FIELD_TYPE.BIT: convert_bit,
FIELD_TYPE.TINY: int,
FIELD_TYPE.SHORT: int,
FIELD_TYPE.LONG: int,
FIELD_TYPE.FLOAT: float,
FIELD_TYPE.DOUBLE: float,
FIELD_TYPE.LONGLONG: int,
FIELD_TYPE.INT24: int,
FIELD_TYPE.YEAR: int,
FIELD_TYPE.TIMESTAMP: convert_datetime,
FIELD_TYPE.DATETIME: convert_datetime,
FIELD_TYPE.TIME: convert_timedelta,
FIELD_TYPE.DATE: convert_date,
FIELD_TYPE.BLOB: through,
FIELD_TYPE.TINY_BLOB: through,
FIELD_TYPE.MEDIUM_BLOB: through,
FIELD_TYPE.LONG_BLOB: through,
FIELD_TYPE.STRING: through,
FIELD_TYPE.VAR_STRING: through,
FIELD_TYPE.VARCHAR: through,
FIELD_TYPE.DECIMAL: Decimal,
FIELD_TYPE.NEWDECIMAL: Decimal,
}
# for MySQLdb compatibility
conversions = encoders.copy()
conversions.update(decoders)
Thing2Literal = escape_str
# Run doctests with `pytest --doctest-modules pymysql/converters.py`
@@ -0,0 +1,515 @@
import re
from . import err
#: Regular expression for :meth:`Cursor.executemany`.
#: executemany only supports simple bulk insert.
#: You can use it to load large dataset.
RE_INSERT_VALUES = re.compile(
r"\s*((?:INSERT|REPLACE)\b.+\bVALUES?\s*)"
+ r"(\(\s*(?:%s|%\([^)]+\)s)\s*(?:,\s*(?:%s|%\([^)]+\)s)\s*)*\))"
+ r'(\s*(?:AS\s+(?:`[^`]+`|"[^"]+"|[0-9A-Za-z_$]+)\s*'
+ r'(?:\(\s*(?:`[^`]+`|"[^"]+"|[0-9A-Za-z_$]+)\s*'
+ r'(?:,\s*(?:`[^`]+`|"[^"]+"|[0-9A-Za-z_$]+)\s*)*\))?\s*)?'
+ r"(?:ON DUPLICATE.*)?);?\s*\Z",
re.IGNORECASE | re.DOTALL,
)
def _backquote_escape(s):
return s.replace("`", "``")
class Cursor:
"""
This is the object used to interact with the database.
Do not create an instance of a Cursor yourself. Call
connections.Connection.cursor().
See `Cursor <https://www.python.org/dev/peps/pep-0249/#cursor-objects>`_ in
the specification.
"""
#: Max statement size which :meth:`executemany` generates.
#:
#: Max size of allowed statement is max_allowed_packet - packet_header_size.
#: Default value of max_allowed_packet is 1048576.
max_stmt_length = 1024000
def __init__(self, connection):
self.connection = connection
self.warning_count = 0
self.description = None
self.rownumber = 0
self.rowcount = -1
self.arraysize = 1
self._executed = None
self._result = None
self._rows = None
def close(self):
"""
Closing a cursor just exhausts all remaining data.
"""
conn = self.connection
if conn is None:
return
try:
while self.nextset():
pass
finally:
self.connection = None
def __enter__(self):
return self
def __exit__(self, *exc_info):
del exc_info
self.close()
def _get_db(self):
if not self.connection:
raise err.ProgrammingError("Cursor closed")
return self.connection
def _check_executed(self):
if not self._executed:
raise err.ProgrammingError("execute() first")
def _conv_row(self, row):
return row
def setinputsizes(self, *args):
"""Does nothing, required by DB API."""
def setoutputsizes(self, *args):
"""Does nothing, required by DB API."""
def _nextset(self, unbuffered=False):
"""Get the next query set."""
conn = self._get_db()
current_result = self._result
if current_result is None or current_result is not conn._result:
return None
if not current_result.has_next:
return None
self._result = None
self._clear_result()
conn.next_result(unbuffered=unbuffered)
self._do_get_result()
return True
def nextset(self):
return self._nextset(False)
def _escape_args(self, args, conn):
if isinstance(args, (tuple, list)):
return tuple(conn.literal(arg) for arg in args)
elif isinstance(args, dict):
return {key: conn.literal(val) for (key, val) in args.items()}
else:
# If it's not a dictionary let's try escaping it anyways.
# Worst case it will throw a Value error
return conn.escape(args)
def mogrify(self, query, args=None):
"""
Returns the exact string that would be sent to the database by calling the
execute() method.
:param query: Query to mogrify.
:type query: str
:param args: Parameters used with query. (optional)
:type args: tuple, list or dict
:return: The query with argument binding applied.
:rtype: str
This method follows the extension to the DB API 2.0 followed by Psycopg.
"""
conn = self._get_db()
if args is not None:
query = query % self._escape_args(args, conn)
return query
def execute(self, query, args=None):
"""Execute a query.
:param query: Query to execute.
:type query: str
:param args: Parameters used with query. (optional)
:type args: tuple, list or dict
:return: Number of affected rows.
:rtype: int
If args is a list or tuple, %s can be used as a placeholder in the query.
If args is a dict, %(name)s can be used as a placeholder in the query.
"""
while self.nextset():
pass
query = self.mogrify(query, args)
result = self._query(query)
self._executed = query
return result
def executemany(self, query, args):
"""Run several data against one query.
:param query: Query to execute.
:type query: str
:param args: Sequence of sequences or mappings. It is used as parameter.
:type args: tuple or list
:return: Number of rows affected, if any.
:rtype: int or None
This method improves performance on multiple-row INSERT and
REPLACE. Otherwise it is equivalent to looping over args with
execute().
"""
if not args:
return
m = RE_INSERT_VALUES.match(query)
if m:
q_prefix = m.group(1) % ()
q_values = m.group(2).rstrip()
q_postfix = m.group(3) or ""
assert q_values[0] == "(" and q_values[-1] == ")"
return self._do_execute_many(
q_prefix,
q_values,
q_postfix,
args,
self.max_stmt_length,
self._get_db().encoding,
)
self.rowcount = sum(self.execute(query, arg) for arg in args)
return self.rowcount
def _do_execute_many(
self, prefix, values, postfix, args, max_stmt_length, encoding
):
conn = self._get_db()
escape = self._escape_args
if isinstance(prefix, str):
prefix = prefix.encode(encoding)
if isinstance(postfix, str):
postfix = postfix.encode(encoding)
sql = bytearray(prefix)
args = iter(args)
v = values % escape(next(args), conn)
if isinstance(v, str):
v = v.encode(encoding, "surrogateescape")
sql += v
rows = 0
for arg in args:
v = values % escape(arg, conn)
if isinstance(v, str):
v = v.encode(encoding, "surrogateescape")
if len(sql) + len(v) + len(postfix) + 1 > max_stmt_length:
rows += self.execute(sql + postfix)
sql = bytearray(prefix)
else:
sql += b","
sql += v
rows += self.execute(sql + postfix)
self.rowcount = rows
return rows
def callproc(self, procname, args=()):
"""Execute stored procedure procname with args.
:param procname: Name of procedure to execute on server.
:type procname: str
:param args: Sequence of parameters to use with procedure.
:type args: tuple or list
Returns the original args.
Compatibility warning: PEP-249 specifies that any modified
parameters must be returned. This is currently impossible
as they are only available by storing them in a server
variable and then retrieved by a query. Since stored
procedures return zero or more result sets, there is no
reliable way to get at OUT or INOUT parameters via callproc.
The server variables are named @_procname_n, where procname
is the parameter above and n is the position of the parameter
(from zero). Once all result sets generated by the procedure
have been fetched, you can issue a SELECT @_procname_0, ...
query using .execute() to get any OUT or INOUT values.
Compatibility warning: The act of calling a stored procedure
itself creates an empty result set. This appears after any
result sets generated by the procedure. This is non-standard
behavior with respect to the DB-API. Be sure to use nextset()
to advance through all result sets; otherwise you may get
disconnected.
"""
procname_escaped = _backquote_escape(procname)
conn = self._get_db()
if args:
fmt = f"@`_{procname_escaped}_%d`=%s"
self._query(
"SET %s"
% ",".join(
fmt % (index, conn.escape(arg)) for index, arg in enumerate(args)
)
)
self.nextset()
q = "CALL `{}`({})".format(
procname_escaped,
",".join([f"@`_{procname_escaped}_{i}`" for i in range(len(args))]),
)
self._query(q)
self._executed = q
return args
def fetchone(self):
"""Fetch the next row."""
self._check_executed()
if self._rows is None or self.rownumber >= len(self._rows):
return None
result = self._rows[self.rownumber]
self.rownumber += 1
return result
def fetchmany(self, size=None):
"""Fetch several rows."""
self._check_executed()
if self._rows is None:
# Django expects () for EOF.
# https://github.com/django/django/blob/0c1518ee429b01c145cf5b34eab01b0b92f8c246/django/db/backends/mysql/features.py#L8
return ()
end = self.rownumber + (size or self.arraysize)
result = self._rows[self.rownumber : end]
self.rownumber = min(end, len(self._rows))
return result
def fetchall(self):
"""Fetch all the rows."""
self._check_executed()
if self._rows is None:
return []
if self.rownumber:
result = self._rows[self.rownumber :]
else:
result = self._rows
self.rownumber = len(self._rows)
return result
def scroll(self, value, mode="relative"):
self._check_executed()
if mode == "relative":
r = self.rownumber + value
elif mode == "absolute":
r = value
else:
raise err.ProgrammingError("unknown scroll mode %s" % mode)
if not (0 <= r < len(self._rows)):
raise IndexError("out of range")
self.rownumber = r
def _query(self, q):
conn = self._get_db()
self._clear_result()
conn.query(q)
self._do_get_result()
return self.rowcount
def _clear_result(self):
self.rownumber = 0
self._result = None
self.rowcount = 0
self.warning_count = 0
self.description = None
self.lastrowid = None
self._rows = None
def _do_get_result(self):
conn = self._get_db()
self._result = result = conn._result
self.rowcount = result.affected_rows
self.warning_count = result.warning_count
self.description = result.description
self.lastrowid = result.insert_id
self._rows = result.rows
def __iter__(self):
return self
def __next__(self):
row = self.fetchone()
if row is None:
raise StopIteration
return row
class DictCursorMixin:
# You can override this to use OrderedDict or other dict-like types.
dict_type = dict
def _do_get_result(self):
super()._do_get_result()
fields = []
if self.description:
for f in self._result.fields:
name = f.name
if name in fields:
name = f.table_name + "." + name
fields.append(name)
self._fields = fields
if fields and self._rows:
self._rows = [self._conv_row(r) for r in self._rows]
def _conv_row(self, row):
if row is None:
return None
return self.dict_type(zip(self._fields, row))
class DictCursor(DictCursorMixin, Cursor):
"""A cursor which returns results as a dictionary"""
class SSCursor(Cursor):
"""
Unbuffered Cursor, mainly useful for queries that return a lot of data,
or for connections to remote servers over a slow network.
Instead of copying every row of data into a buffer, this will fetch
rows as needed. The upside of this is the client uses much less memory,
and rows are returned much faster when traveling over a slow network
or if the result set is very big.
There are limitations, though. The MySQL protocol doesn't support
returning the total number of rows, so the only way to tell how many rows
there are is to iterate over every row returned. Also, it currently isn't
possible to scroll backwards, as only the current row is held in memory.
"""
def _conv_row(self, row):
return row
def close(self):
conn = self.connection
if conn is None:
return
if self._result is not None and self._result is conn._result:
self._result._finish_unbuffered_query()
try:
while self.nextset():
pass
finally:
self.connection = None
__del__ = close
def _query(self, q):
conn = self._get_db()
self._clear_result()
conn.query(q, unbuffered=True)
self._do_get_result()
return self.rowcount
def nextset(self):
return self._nextset(unbuffered=True)
def read_next(self):
"""Read next row."""
return self._conv_row(self._result._read_rowdata_packet_unbuffered())
def fetchone(self):
"""Fetch next row."""
self._check_executed()
row = self.read_next()
if row is None:
self.warning_count = self._result.warning_count
return None
self.rownumber += 1
return row
def fetchall(self):
"""
Fetch all, as per MySQLdb. Pretty useless for large queries, as
it is buffered. See fetchall_unbuffered(), if you want an unbuffered
generator version of this method.
"""
return list(self.fetchall_unbuffered())
def fetchall_unbuffered(self):
"""
Fetch all, implemented as a generator, which isn't to standard,
however, it doesn't make sense to return everything in a list, as that
would use ridiculous memory for large result sets.
"""
return iter(self.fetchone, None)
def fetchmany(self, size=None):
"""Fetch many."""
self._check_executed()
if size is None:
size = self.arraysize
rows = []
for i in range(size):
row = self.read_next()
if row is None:
self.warning_count = self._result.warning_count
break
rows.append(row)
self.rownumber += 1
if not rows:
# Django expects () for EOF.
# https://github.com/django/django/blob/0c1518ee429b01c145cf5b34eab01b0b92f8c246/django/db/backends/mysql/features.py#L8
return ()
return rows
def scroll(self, value, mode="relative"):
self._check_executed()
if mode == "relative":
if value < 0:
raise err.NotSupportedError(
"Backwards scrolling not supported by this cursor"
)
for _ in range(value):
self.read_next()
self.rownumber += value
elif mode == "absolute":
if value < self.rownumber:
raise err.NotSupportedError(
"Backwards scrolling not supported by this cursor"
)
end = value - self.rownumber
for _ in range(end):
self.read_next()
self.rownumber = value
else:
raise err.ProgrammingError("unknown scroll mode %s" % mode)
class SSDictCursor(DictCursorMixin, SSCursor):
"""An unbuffered cursor, which returns results as a dictionary"""
@@ -0,0 +1,154 @@
import struct
from .constants import ER
class MySQLError(Exception):
"""Exception related to operation with MySQL."""
class Warning(Warning, MySQLError):
"""Exception raised for important warnings like data truncations
while inserting, etc."""
class Error(MySQLError):
"""Exception that is the base class of all other error exceptions
(not Warning)."""
def __init__(self, *args, sqlstate=None):
super().__init__(*args)
self.sqlstate = sqlstate
class InterfaceError(Error):
"""Exception raised for errors that are related to the database
interface rather than the database itself."""
class DatabaseError(Error):
"""Exception raised for errors that are related to the
database."""
class DataError(DatabaseError):
"""Exception raised for errors that are due to problems with the
processed data like division by zero, numeric value out of range,
etc."""
class OperationalError(DatabaseError):
"""Exception raised for errors that are related to the database's
operation and not necessarily under the control of the programmer,
e.g. an unexpected disconnect occurs, the data source name is not
found, a transaction could not be processed, a memory allocation
error occurred during processing, etc."""
class IntegrityError(DatabaseError):
"""Exception raised when the relational integrity of the database
is affected, e.g. a foreign key check fails, duplicate key,
etc."""
class InternalError(DatabaseError):
"""Exception raised when the database encounters an internal
error, e.g. the cursor is not valid anymore, the transaction is
out of sync, etc."""
class ProgrammingError(DatabaseError):
"""Exception raised for programming errors, e.g. table not found
or already exists, syntax error in the SQL statement, wrong number
of parameters specified, etc."""
class NotSupportedError(DatabaseError):
"""Exception raised in case a method or database API was used
which is not supported by the database, e.g. requesting a
.rollback() on a connection that does not support transaction or
has transactions turned off."""
error_map = {}
def _map_error(exc, *errors):
for error in errors:
error_map[error] = exc
_map_error(
ProgrammingError,
ER.DB_CREATE_EXISTS,
ER.SYNTAX_ERROR,
ER.PARSE_ERROR,
ER.NO_SUCH_TABLE,
ER.WRONG_DB_NAME,
ER.WRONG_TABLE_NAME,
ER.FIELD_SPECIFIED_TWICE,
ER.INVALID_GROUP_FUNC_USE,
ER.UNSUPPORTED_EXTENSION,
ER.TABLE_MUST_HAVE_COLUMNS,
ER.CANT_DO_THIS_DURING_AN_TRANSACTION,
ER.WRONG_DB_NAME,
ER.WRONG_COLUMN_NAME,
)
_map_error(
DataError,
ER.WARN_DATA_TRUNCATED,
ER.WARN_NULL_TO_NOTNULL,
ER.WARN_DATA_OUT_OF_RANGE,
ER.NO_DEFAULT,
ER.PRIMARY_CANT_HAVE_NULL,
ER.DATA_TOO_LONG,
ER.DATETIME_FUNCTION_OVERFLOW,
ER.TRUNCATED_WRONG_VALUE_FOR_FIELD,
ER.ILLEGAL_VALUE_FOR_TYPE,
)
_map_error(
IntegrityError,
ER.DUP_ENTRY,
ER.NO_REFERENCED_ROW,
ER.NO_REFERENCED_ROW_2,
ER.ROW_IS_REFERENCED,
ER.ROW_IS_REFERENCED_2,
ER.CANNOT_ADD_FOREIGN,
ER.BAD_NULL_ERROR,
)
_map_error(
NotSupportedError,
ER.WARNING_NOT_COMPLETE_ROLLBACK,
ER.NOT_SUPPORTED_YET,
ER.FEATURE_DISABLED,
ER.UNKNOWN_STORAGE_ENGINE,
)
_map_error(
OperationalError,
ER.DBACCESS_DENIED_ERROR,
ER.ACCESS_DENIED_ERROR,
ER.CON_COUNT_ERROR,
ER.TABLEACCESS_DENIED_ERROR,
ER.COLUMNACCESS_DENIED_ERROR,
ER.CONSTRAINT_FAILED,
ER.LOCK_DEADLOCK,
)
del _map_error, ER
def raise_mysql_exception(data):
errno = struct.unpack("<h", data[1:3])[0]
# https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_err_packet.html
# Error packet has optional sqlstate that is 5 bytes and starts with '#'.
sqlstate = None
if data[3] == 0x23: # '#'
sqlstate = data[4:9].decode()
errval = data[9:].decode("utf-8", "replace")
else:
errval = data[3:].decode("utf-8", "replace")
errorclass = error_map.get(errno)
if errorclass is None:
errorclass = InternalError if errno < 1000 else OperationalError
raise errorclass(errno, errval, sqlstate=sqlstate)
@@ -0,0 +1,21 @@
import configparser
class Parser(configparser.RawConfigParser):
def __init__(self, **kwargs):
kwargs["allow_no_value"] = True
configparser.RawConfigParser.__init__(self, **kwargs)
def __remove_quotes(self, value):
quotes = ["'", '"']
for quote in quotes:
if len(value) >= 2 and value[0] == value[-1] == quote:
return value[1:-1]
return value
def optionxform(self, key):
return key.lower().replace("_", "-")
def get(self, section, option):
value = configparser.RawConfigParser.get(self, section, option)
return self.__remove_quotes(value)
@@ -0,0 +1,356 @@
# Python implementation of low level MySQL client-server protocol
# http://dev.mysql.com/doc/internals/en/client-server-protocol.html
from .charset import MBLENGTH
from .constants import FIELD_TYPE, SERVER_STATUS
from . import err
import struct
import sys
DEBUG = False
NULL_COLUMN = 251
UNSIGNED_CHAR_COLUMN = 251
UNSIGNED_SHORT_COLUMN = 252
UNSIGNED_INT24_COLUMN = 253
UNSIGNED_INT64_COLUMN = 254
def dump_packet(data): # pragma: no cover
def printable(data):
if 32 <= data < 127:
return chr(data)
return "."
try:
print("packet length:", len(data))
for i in range(1, 7):
f = sys._getframe(i)
print("call[%d]: %s (line %d)" % (i, f.f_code.co_name, f.f_lineno))
print("-" * 66)
except ValueError:
pass
dump_data = [data[i : i + 16] for i in range(0, min(len(data), 256), 16)]
for d in dump_data:
print(
" ".join(f"{x:02X}" for x in d)
+ " " * (16 - len(d))
+ " " * 2
+ "".join(printable(x) for x in d)
)
print("-" * 66)
print()
class MysqlPacket:
"""Representation of a MySQL response packet.
Provides an interface for reading/parsing the packet results.
"""
__slots__ = ("_position", "_data")
def __init__(self, data, encoding):
self._position = 0
self._data = data
def get_all_data(self):
return self._data
def read(self, size):
"""Read the first 'size' bytes in packet and advance cursor past them."""
result = self._data[self._position : (self._position + size)]
if len(result) != size:
error = (
"Result length not requested length:\n"
f"Expected={size}. Actual={len(result)}. Position: {self._position}. Data Length: {len(self._data)}"
)
if DEBUG:
print(error)
self.dump()
raise AssertionError(error)
self._position += size
return result
def read_all(self):
"""Read all remaining data in the packet.
(Subsequent read() will return errors.)
"""
result = self._data[self._position :]
self._position = None # ensure no subsequent read()
return result
def advance(self, length):
"""Advance the cursor in data buffer 'length' bytes."""
new_position = self._position + length
if new_position < 0 or new_position > len(self._data):
raise Exception(
f"Invalid advance amount ({length}) for cursor. Position={new_position}"
)
self._position = new_position
def rewind(self, position=0):
"""Set the position of the data buffer cursor to 'position'."""
if position < 0 or position > len(self._data):
raise Exception("Invalid position to rewind cursor to: %s." % position)
self._position = position
def get_bytes(self, position, length=1):
"""Get 'length' bytes starting at 'position'.
Position is start of payload (first four packet header bytes are not
included) starting at index '0'.
No error checking is done. If requesting outside end of buffer
an empty string (or string shorter than 'length') may be returned!
"""
return self._data[position : (position + length)]
def read_uint8(self):
result = self._data[self._position]
self._position += 1
return result
def read_uint16(self):
result = struct.unpack_from("<H", self._data, self._position)[0]
self._position += 2
return result
def read_uint24(self):
low, high = struct.unpack_from("<HB", self._data, self._position)
self._position += 3
return low + (high << 16)
def read_uint32(self):
result = struct.unpack_from("<I", self._data, self._position)[0]
self._position += 4
return result
def read_uint64(self):
result = struct.unpack_from("<Q", self._data, self._position)[0]
self._position += 8
return result
def read_string(self):
end_pos = self._data.find(b"\0", self._position)
if end_pos < 0:
return None
result = self._data[self._position : end_pos]
self._position = end_pos + 1
return result
def read_length_encoded_integer(self):
"""Read a 'Length Coded Binary' number from the data buffer.
Length coded numbers can be anywhere from 1 to 9 bytes depending
on the value of the first byte.
"""
c = self.read_uint8()
if c == NULL_COLUMN:
return None
if c < UNSIGNED_CHAR_COLUMN:
return c
elif c == UNSIGNED_SHORT_COLUMN:
return self.read_uint16()
elif c == UNSIGNED_INT24_COLUMN:
return self.read_uint24()
elif c == UNSIGNED_INT64_COLUMN:
return self.read_uint64()
def read_length_coded_string(self):
"""Read a 'Length Coded String' from the data buffer.
A 'Length Coded String' consists first of a length coded
(unsigned, positive) integer represented in 1-9 bytes followed by
that many bytes of binary data. (For example "cat" would be "3cat".)
"""
length = self.read_length_encoded_integer()
if length is None:
return None
return self.read(length)
def read_struct(self, fmt):
s = struct.Struct(fmt)
result = s.unpack_from(self._data, self._position)
self._position += s.size
return result
def is_ok_packet(self):
# https://dev.mysql.com/doc/internals/en/packet-OK_Packet.html
return self._data[0] == 0 and len(self._data) >= 7
def is_eof_packet(self):
# http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-EOF_Packet
# Caution: \xFE may be LengthEncodedInteger.
# If \xFE is LengthEncodedInteger header, 8bytes followed.
return self._data[0] == 0xFE and len(self._data) < 9
def is_auth_switch_request(self):
# http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchRequest
return self._data[0] == 0xFE
def is_extra_auth_data(self):
# https://dev.mysql.com/doc/internals/en/successful-authentication.html
return self._data[0] == 1
def is_resultset_packet(self):
field_count = self._data[0]
return 1 <= field_count <= 250
def is_load_local_packet(self):
return self._data[0] == 0xFB
def is_error_packet(self):
return self._data[0] == 0xFF
def check_error(self):
if self.is_error_packet():
self.raise_for_error()
def raise_for_error(self):
self.rewind()
self.advance(1) # field_count == error (we already know that)
errno = self.read_uint16()
if DEBUG:
print("errno =", errno)
err.raise_mysql_exception(self._data)
def dump(self):
dump_packet(self._data)
class FieldDescriptorPacket(MysqlPacket):
"""A MysqlPacket that represents a specific column's metadata in the result.
Parsing is automatically done and the results are exported via public
attributes on the class such as: db, table_name, name, length, type_code.
"""
def __init__(self, data, encoding):
MysqlPacket.__init__(self, data, encoding)
self._parse_field_descriptor(encoding)
def _parse_field_descriptor(self, encoding):
"""Parse the 'Field Descriptor' (Metadata) packet.
This is compatible with MySQL 4.1+ (not compatible with MySQL 4.0).
"""
self.catalog = self.read_length_coded_string()
self.db = self.read_length_coded_string()
self.table_name = self.read_length_coded_string().decode(encoding)
self.org_table = self.read_length_coded_string().decode(encoding)
self.name = self.read_length_coded_string().decode(encoding)
self.org_name = self.read_length_coded_string().decode(encoding)
(
self.charsetnr,
self.length,
self.type_code,
self.flags,
self.scale,
) = self.read_struct("<xHIBHBxx")
# 'default' is a length coded binary and is still in the buffer?
# not used for normal result sets...
def description(self):
"""Provides a 7-item tuple compatible with the Python PEP249 DB Spec."""
return (
self.name,
self.type_code,
None, # TODO: display_length; should this be self.length?
self.get_column_length(), # 'internal_size'
self.get_column_length(), # 'precision' # TODO: why!?!?
self.scale,
self.flags % 2 == 0,
)
def get_column_length(self):
if self.type_code == FIELD_TYPE.VAR_STRING:
mblen = MBLENGTH.get(self.charsetnr, 1)
return self.length // mblen
return self.length
def __str__(self):
return "{} {!r}.{!r}.{!r}, type={}, flags={:x}".format(
self.__class__,
self.db,
self.table_name,
self.name,
self.type_code,
self.flags,
)
class OKPacketWrapper:
"""
OK Packet Wrapper. It uses an existing packet object, and wraps
around it, exposing useful variables while still providing access
to the original packet objects variables and methods.
"""
def __init__(self, from_packet):
if not from_packet.is_ok_packet():
raise ValueError(
"Cannot create "
+ str(self.__class__.__name__)
+ " object from invalid packet type"
)
self.packet = from_packet
self.packet.advance(1)
self.affected_rows = self.packet.read_length_encoded_integer()
self.insert_id = self.packet.read_length_encoded_integer()
self.server_status, self.warning_count = self.read_struct("<HH")
self.message = self.packet.read_all()
self.has_next = self.server_status & SERVER_STATUS.SERVER_MORE_RESULTS_EXISTS
def __getattr__(self, key):
return getattr(self.packet, key)
class EOFPacketWrapper:
"""
EOF Packet Wrapper. It uses an existing packet object, and wraps
around it, exposing useful variables while still providing access
to the original packet objects variables and methods.
"""
def __init__(self, from_packet):
if not from_packet.is_eof_packet():
raise ValueError(
f"Cannot create '{self.__class__}' object from invalid packet type"
)
self.packet = from_packet
self.warning_count, self.server_status = self.packet.read_struct("<xhh")
if DEBUG:
print("server_status=", self.server_status)
self.has_next = self.server_status & SERVER_STATUS.SERVER_MORE_RESULTS_EXISTS
def __getattr__(self, key):
return getattr(self.packet, key)
class LoadLocalPacketWrapper:
"""
Load Local Packet Wrapper. It uses an existing packet object, and wraps
around it, exposing useful variables while still providing access
to the original packet objects variables and methods.
"""
def __init__(self, from_packet):
if not from_packet.is_load_local_packet():
raise ValueError(
f"Cannot create '{self.__class__}' object from invalid packet type"
)
self.packet = from_packet
self.filename = self.packet.get_all_data()[1:]
if DEBUG:
print("filename=", self.filename)
def __getattr__(self, key):
return getattr(self.packet, key)
@@ -0,0 +1,20 @@
from time import localtime
from datetime import date, datetime, time, timedelta
Date = date
Time = time
TimeDelta = timedelta
Timestamp = datetime
def DateFromTicks(ticks):
return date(*localtime(ticks)[:3])
def TimeFromTicks(ticks):
return time(*localtime(ticks)[3:6])
def TimestampFromTicks(ticks):
return datetime(*localtime(ticks)[:6])
@@ -0,0 +1 @@
pip
@@ -0,0 +1,76 @@
There are two licenses associated with xlrd. This one relates to the bulk of
the work done on the library::
Portions copyright © 2005-2009, Stephen John Machin, Lingfo Pty Ltd
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. None of the names of Stephen John Machin, Lingfo Pty Ltd and any
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
THE POSSIBILITY OF SUCH DAMAGE.
This one covers some earlier work::
/*-
* Copyright (c) 2001 David Giffin.
* All rights reserved.
*
* Based on the the Java version: Andrew Khan Copyright (c) 2000.
*
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* 3. All advertising materials mentioning features or use of this
* software must display the following acknowledgment:
* "This product includes software developed by
* David Giffin <david@giffin.org>."
*
* 4. Redistributions of any form whatsoever must retain the following
* acknowledgment:
* "This product includes software developed by
* David Giffin <david@giffin.org>."
*
* THIS SOFTWARE IS PROVIDED BY DAVID GIFFIN ``AS IS'' AND ANY
* EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL DAVID GIFFIN OR
* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
* OF THE POSSIBILITY OF SUCH DAMAGE.
*/
@@ -0,0 +1,98 @@
Metadata-Version: 2.1
Name: xlrd
Version: 2.0.2
Summary: Library for developers to extract data from Microsoft Excel (tm) .xls spreadsheet files
Home-page: http://www.python-excel.org/
Author: Chris Withers
Author-email: chris@withers.org
License: BSD
Keywords: xls,excel,spreadsheet,workbook
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: BSD License
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Operating System :: OS Independent
Classifier: Topic :: Database
Classifier: Topic :: Office/Business
Classifier: Topic :: Software Development :: Libraries :: Python Modules
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*
License-File: LICENSE
Provides-Extra: build
Requires-Dist: wheel ; extra == 'build'
Requires-Dist: twine ; extra == 'build'
Provides-Extra: docs
Requires-Dist: sphinx ; extra == 'docs'
Provides-Extra: test
Requires-Dist: pytest ; extra == 'test'
Requires-Dist: pytest-cov ; extra == 'test'
xlrd
====
|Build Status|_ |Coverage Status|_ |Documentation|_ |PyPI version|_
.. |Build Status| image:: https://circleci.com/gh/python-excel/xlrd/tree/master.svg?style=shield
.. _Build Status: https://circleci.com/gh/python-excel/xlrd/tree/master
.. |Coverage Status| image:: https://codecov.io/gh/python-excel/xlrd/branch/master/graph/badge.svg?token=lNSqwBBbvk
.. _Coverage Status: https://codecov.io/gh/python-excel/xlrd
.. |Documentation| image:: https://readthedocs.org/projects/xlrd/badge/?version=latest
.. _Documentation: http://xlrd.readthedocs.io/en/latest/?badge=latest
.. |PyPI version| image:: https://badge.fury.io/py/xlrd.svg
.. _PyPI version: https://badge.fury.io/py/xlrd
xlrd is a library for reading data and formatting information from Excel
files in the historical ``.xls`` format.
.. warning::
This library will no longer read anything other than ``.xls`` files. For
alternatives that read newer file formats, please see http://www.python-excel.org/.
The following are also not supported but will safely and reliably be ignored:
* Charts, Macros, Pictures, any other embedded object, **including** embedded worksheets.
* VBA modules
* Formulas, but results of formula calculations are extracted.
* Comments
* Hyperlinks
* Autofilters, advanced filters, pivot tables, conditional formatting, data validation
Password-protected files are not supported and cannot be read by this library.
Quick start:
.. code-block:: bash
pip install xlrd
.. code-block:: python
import xlrd
book = xlrd.open_workbook("myfile.xls")
print("The number of worksheets is {0}".format(book.nsheets))
print("Worksheet name(s): {0}".format(book.sheet_names()))
sh = book.sheet_by_index(0)
print("{0} {1} {2}".format(sh.name, sh.nrows, sh.ncols))
print("Cell D30 is {0}".format(sh.cell_value(rowx=29, colx=3)))
for rx in range(sh.nrows):
print(sh.row(rx))
From the command line, this will show the first, second and last rows of each sheet in each file:
.. code-block:: bash
python PYDIR/scripts/runxlrd.py 3rows *blah*.xls
@@ -0,0 +1,29 @@
../../bin/__pycache__/runxlrd.cpython-312.pyc,,
../../bin/runxlrd.py,sha256=TcwCvUPpY8g31UVq3GV_dNx64IXQkPxPHI2T0SaXFDE,16144
xlrd-2.0.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
xlrd-2.0.2.dist-info/LICENSE,sha256=taXbzmAmXjBagVpsuD7QfyRRnYumRPKjB5lEiLztiBU,3771
xlrd-2.0.2.dist-info/METADATA,sha256=riY9uk1hYhC7LXCc0JG6RTmEMU95-PUPRK-GQ1Jlbew,3513
xlrd-2.0.2.dist-info/RECORD,,
xlrd-2.0.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
xlrd-2.0.2.dist-info/WHEEL,sha256=WzZ8cwjh8l0jtULNjYq1Hpr-WCqCRgPr--TX4P5I1Wo,110
xlrd-2.0.2.dist-info/top_level.txt,sha256=nZ-t3Sc_CqNsctS3L-V36qXwC5DuAFvkB1inJApF_vM,5
xlrd/__init__.py,sha256=kmuIEz8cutatGjjW3EmaApTEgcHmcsYo0CcWH_th2P0,7320
xlrd/__pycache__/__init__.cpython-312.pyc,,
xlrd/__pycache__/biffh.cpython-312.pyc,,
xlrd/__pycache__/book.cpython-312.pyc,,
xlrd/__pycache__/compdoc.cpython-312.pyc,,
xlrd/__pycache__/formatting.cpython-312.pyc,,
xlrd/__pycache__/formula.cpython-312.pyc,,
xlrd/__pycache__/info.cpython-312.pyc,,
xlrd/__pycache__/sheet.cpython-312.pyc,,
xlrd/__pycache__/timemachine.cpython-312.pyc,,
xlrd/__pycache__/xldate.cpython-312.pyc,,
xlrd/biffh.py,sha256=GjtBoI9PNIEdbZmGTvfDQ8HuhZiLJd2ofdo2qmunzmA,16651
xlrd/book.py,sha256=1LHxyWMb50ZBIwUlZC4UtIFem5y8o8M3zADcdPCF_84,57527
xlrd/compdoc.py,sha256=3xkfuJbLRGRMdPiZVBsWAgGktw7Jna-w3p_lcTCT6YQ,21091
xlrd/formatting.py,sha256=6Mh6GOM7FZTj0cJ4FHSl0CVo-8InYDv2KIZTMycEw6Q,45573
xlrd/formula.py,sha256=cgEXNWCaYVKzIZwa7IW0tDcC1IXzUvhqJXUOL75Xoaw,94460
xlrd/info.py,sha256=AwsvL57Z-lmKDDaCBfUVokwMwPvJLn9jjGtqP9gevOw,36
xlrd/sheet.py,sha256=mhwEqEteUIQY9jK5CyX87stuk2MmXURONf86kZQMhY0,106806
xlrd/timemachine.py,sha256=7CtU7FY6mQW7HSP5FjM1K9pIfNexVCt_PJXG2M3FEAc,1757
xlrd/xldate.py,sha256=S-uUaLX8kauGy2SkX_M-S-FBqqQ6H9EfTAVpv5FMX0I,7934
@@ -0,0 +1,6 @@
Wheel-Version: 1.0
Generator: bdist_wheel (0.37.0)
Root-Is-Purelib: true
Tag: py2-none-any
Tag: py3-none-any
@@ -0,0 +1,213 @@
# Copyright (c) 2005-2012 Stephen John Machin, Lingfo Pty Ltd
# This module is part of the xlrd package, which is released under a
# BSD-style licence.
import os
import pprint
import sys
import zipfile
from . import timemachine
from .biffh import (
XL_CELL_BLANK, XL_CELL_BOOLEAN, XL_CELL_DATE, XL_CELL_EMPTY, XL_CELL_ERROR,
XL_CELL_NUMBER, XL_CELL_TEXT, XLRDError, biff_text_from_num,
error_text_from_code,
)
from .book import Book, colname, open_workbook_xls
from .compdoc import SIGNATURE as XLS_SIGNATURE
from .formula import * # is constrained by __all__
from .info import __VERSION__, __version__
from .sheet import empty_cell
from .xldate import XLDateError, xldate_as_datetime, xldate_as_tuple
#: descriptions of the file types :mod:`xlrd` can :func:`inspect <inspect_format>`.
FILE_FORMAT_DESCRIPTIONS = {
'xls': 'Excel xls',
'xlsb': 'Excel 2007 xlsb file',
'xlsx': 'Excel xlsx file',
'ods': 'Openoffice.org ODS file',
'zip': 'Unknown ZIP file',
None: 'Unknown file type',
}
ZIP_SIGNATURE = b"PK\x03\x04"
PEEK_SIZE = max(len(XLS_SIGNATURE), len(ZIP_SIGNATURE))
def inspect_format(path=None, content=None):
"""
Inspect the content at the supplied path or the :class:`bytes` content provided
and return the file's type as a :class:`str`, or ``None`` if it cannot
be determined.
:param path:
A :class:`string <str>` path containing the content to inspect.
``~`` will be expanded.
:param content:
The :class:`bytes` content to inspect.
:returns:
A :class:`str`, or ``None`` if the format cannot be determined.
The return value can always be looked up in :data:`FILE_FORMAT_DESCRIPTIONS`
to return a human-readable description of the format found.
"""
if content:
peek = content[:PEEK_SIZE]
else:
path = os.path.expanduser(path)
with open(path, "rb") as f:
peek = f.read(PEEK_SIZE)
if peek.startswith(XLS_SIGNATURE):
return 'xls'
if peek.startswith(ZIP_SIGNATURE):
zf = zipfile.ZipFile(timemachine.BYTES_IO(content) if content else path)
# Workaround for some third party files that use forward slashes and
# lower case names. We map the expected name in lowercase to the
# actual filename in the zip container.
component_names = {name.replace('\\', '/').lower(): name
for name in zf.namelist()}
if 'xl/workbook.xml' in component_names:
return 'xlsx'
if 'xl/workbook.bin' in component_names:
return 'xlsb'
if 'content.xml' in component_names:
return 'ods'
return 'zip'
def open_workbook(filename=None,
logfile=sys.stdout,
verbosity=0,
use_mmap=True,
file_contents=None,
encoding_override=None,
formatting_info=False,
on_demand=False,
ragged_rows=False,
ignore_workbook_corruption=False
):
"""
Open a spreadsheet file for data extraction.
:param filename: The path to the spreadsheet file to be opened.
:param logfile: An open file to which messages and diagnostics are written.
:param verbosity: Increases the volume of trace material written to the
logfile.
:param use_mmap:
Whether to use the mmap module is determined heuristically.
Use this arg to override the result.
Current heuristic: mmap is used if it exists.
:param file_contents:
A string or an :class:`mmap.mmap` object or some other behave-alike
object. If ``file_contents`` is supplied, ``filename`` will not be used,
except (possibly) in messages.
:param encoding_override:
Used to overcome missing or bad codepage information
in older-version files. See :doc:`unicode`.
:param formatting_info:
The default is ``False``, which saves memory.
In this case, "Blank" cells, which are those with their own formatting
information but no data, are treated as empty by ignoring the file's
``BLANK`` and ``MULBLANK`` records.
This cuts off any bottom or right "margin" of rows of empty or blank
cells.
Only :meth:`~xlrd.sheet.Sheet.cell_value` and
:meth:`~xlrd.sheet.Sheet.cell_type` are available.
When ``True``, formatting information will be read from the spreadsheet
file. This provides all cells, including empty and blank cells.
Formatting information is available for each cell.
Note that this will raise a NotImplementedError when used with an
xlsx file.
:param on_demand:
Governs whether sheets are all loaded initially or when demanded
by the caller. See :doc:`on_demand`.
:param ragged_rows:
The default of ``False`` means all rows are padded out with empty cells so
that all rows have the same size as found in
:attr:`~xlrd.sheet.Sheet.ncols`.
``True`` means that there are no empty cells at the ends of rows.
This can result in substantial memory savings if rows are of widely
varying sizes. See also the :meth:`~xlrd.sheet.Sheet.row_len` method.
:param ignore_workbook_corruption:
This option allows to read corrupted workbooks.
When ``False`` you may face CompDocError: Workbook corruption.
When ``True`` that exception will be ignored.
:returns: An instance of the :class:`~xlrd.book.Book` class.
"""
file_format = inspect_format(filename, file_contents)
# We have to let unknown file formats pass through here, as some ancient
# files that xlrd can parse don't start with the expected signature.
if file_format and file_format != 'xls':
raise XLRDError(FILE_FORMAT_DESCRIPTIONS[file_format]+'; not supported')
bk = open_workbook_xls(
filename=filename,
logfile=logfile,
verbosity=verbosity,
use_mmap=use_mmap,
file_contents=file_contents,
encoding_override=encoding_override,
formatting_info=formatting_info,
on_demand=on_demand,
ragged_rows=ragged_rows,
ignore_workbook_corruption=ignore_workbook_corruption,
)
return bk
def dump(filename, outfile=sys.stdout, unnumbered=False):
"""
For debugging: dump an XLS file's BIFF records in char & hex.
:param filename: The path to the file to be dumped.
:param outfile: An open file, to which the dump is written.
:param unnumbered: If true, omit offsets (for meaningful diffs).
"""
from .biffh import biff_dump
bk = Book()
bk.biff2_8_load(filename=filename, logfile=outfile, )
biff_dump(bk.mem, bk.base, bk.stream_len, 0, outfile, unnumbered)
def count_records(filename, outfile=sys.stdout):
"""
For debugging and analysis: summarise the file's BIFF records.
ie: produce a sorted file of ``(record_name, count)``.
:param filename: The path to the file to be summarised.
:param outfile: An open file, to which the summary is written.
"""
from .biffh import biff_count_records
bk = Book()
bk.biff2_8_load(filename=filename, logfile=outfile, )
biff_count_records(bk.mem, bk.base, bk.stream_len, outfile)
@@ -0,0 +1,643 @@
# -*- coding: utf-8 -*-
# Portions copyright © 2005-2010 Stephen John Machin, Lingfo Pty Ltd
# This module is part of the xlrd package, which is released under a
# BSD-style licence.
from __future__ import print_function
import sys
from struct import unpack
from .timemachine import *
DEBUG = 0
class XLRDError(Exception):
"""
An exception indicating problems reading data from an Excel file.
"""
class BaseObject(object):
"""
Parent of almost all other classes in the package. Defines a common
:meth:`dump` method for debugging.
"""
_repr_these = []
def dump(self, f=None, header=None, footer=None, indent=0):
"""
:param f: open file object, to which the dump is written
:param header: text to write before the dump
:param footer: text to write after the dump
:param indent: number of leading spaces (for recursive calls)
"""
if f is None:
f = sys.stderr
if hasattr(self, "__slots__"):
alist = []
for attr in self.__slots__:
alist.append((attr, getattr(self, attr)))
else:
alist = self.__dict__.items()
alist = sorted(alist)
pad = " " * indent
if header is not None: print(header, file=f)
list_type = type([])
dict_type = type({})
for attr, value in alist:
if getattr(value, 'dump', None) and attr != 'book':
value.dump(f,
header="%s%s (%s object):" % (pad, attr, value.__class__.__name__),
indent=indent+4)
elif (attr not in self._repr_these and
(isinstance(value, list_type) or isinstance(value, dict_type))):
print("%s%s: %s, len = %d" % (pad, attr, type(value), len(value)), file=f)
else:
fprintf(f, "%s%s: %r\n", pad, attr, value)
if footer is not None: print(footer, file=f)
FUN, FDT, FNU, FGE, FTX = range(5) # unknown, date, number, general, text
DATEFORMAT = FDT
NUMBERFORMAT = FNU
(
XL_CELL_EMPTY,
XL_CELL_TEXT,
XL_CELL_NUMBER,
XL_CELL_DATE,
XL_CELL_BOOLEAN,
XL_CELL_ERROR,
XL_CELL_BLANK, # for use in debugging, gathering stats, etc
) = range(7)
biff_text_from_num = {
0: "(not BIFF)",
20: "2.0",
21: "2.1",
30: "3",
40: "4S",
45: "4W",
50: "5",
70: "7",
80: "8",
85: "8X",
}
#: This dictionary can be used to produce a text version of the internal codes
#: that Excel uses for error cells.
error_text_from_code = {
0x00: '#NULL!', # Intersection of two cell ranges is empty
0x07: '#DIV/0!', # Division by zero
0x0F: '#VALUE!', # Wrong type of operand
0x17: '#REF!', # Illegal or deleted cell reference
0x1D: '#NAME?', # Wrong function or range name
0x24: '#NUM!', # Value range overflow
0x2A: '#N/A', # Argument or function not available
}
BIFF_FIRST_UNICODE = 80
XL_WORKBOOK_GLOBALS = WBKBLOBAL = 0x5
XL_WORKBOOK_GLOBALS_4W = 0x100
XL_WORKSHEET = WRKSHEET = 0x10
XL_BOUNDSHEET_WORKSHEET = 0x00
XL_BOUNDSHEET_CHART = 0x02
XL_BOUNDSHEET_VB_MODULE = 0x06
# XL_RK2 = 0x7e
XL_ARRAY = 0x0221
XL_ARRAY2 = 0x0021
XL_BLANK = 0x0201
XL_BLANK_B2 = 0x01
XL_BOF = 0x809
XL_BOOLERR = 0x205
XL_BOOLERR_B2 = 0x5
XL_BOUNDSHEET = 0x85
XL_BUILTINFMTCOUNT = 0x56
XL_CF = 0x01B1
XL_CODEPAGE = 0x42
XL_COLINFO = 0x7D
XL_COLUMNDEFAULT = 0x20 # BIFF2 only
XL_COLWIDTH = 0x24 # BIFF2 only
XL_CONDFMT = 0x01B0
XL_CONTINUE = 0x3c
XL_COUNTRY = 0x8C
XL_DATEMODE = 0x22
XL_DEFAULTROWHEIGHT = 0x0225
XL_DEFCOLWIDTH = 0x55
XL_DIMENSION = 0x200
XL_DIMENSION2 = 0x0
XL_EFONT = 0x45
XL_EOF = 0x0a
XL_EXTERNNAME = 0x23
XL_EXTERNSHEET = 0x17
XL_EXTSST = 0xff
XL_FEAT11 = 0x872
XL_FILEPASS = 0x2f
XL_FONT = 0x31
XL_FONT_B3B4 = 0x231
XL_FORMAT = 0x41e
XL_FORMAT2 = 0x1E # BIFF2, BIFF3
XL_FORMULA = 0x6
XL_FORMULA3 = 0x206
XL_FORMULA4 = 0x406
XL_GCW = 0xab
XL_HLINK = 0x01B8
XL_QUICKTIP = 0x0800
XL_HORIZONTALPAGEBREAKS = 0x1b
XL_INDEX = 0x20b
XL_INTEGER = 0x2 # BIFF2 only
XL_IXFE = 0x44 # BIFF2 only
XL_LABEL = 0x204
XL_LABEL_B2 = 0x04
XL_LABELRANGES = 0x15f
XL_LABELSST = 0xfd
XL_LEFTMARGIN = 0x26
XL_TOPMARGIN = 0x28
XL_RIGHTMARGIN = 0x27
XL_BOTTOMMARGIN = 0x29
XL_HEADER = 0x14
XL_FOOTER = 0x15
XL_HCENTER = 0x83
XL_VCENTER = 0x84
XL_MERGEDCELLS = 0xE5
XL_MSO_DRAWING = 0x00EC
XL_MSO_DRAWING_GROUP = 0x00EB
XL_MSO_DRAWING_SELECTION = 0x00ED
XL_MULRK = 0xbd
XL_MULBLANK = 0xbe
XL_NAME = 0x18
XL_NOTE = 0x1c
XL_NUMBER = 0x203
XL_NUMBER_B2 = 0x3
XL_OBJ = 0x5D
XL_PAGESETUP = 0xA1
XL_PALETTE = 0x92
XL_PANE = 0x41
XL_PRINTGRIDLINES = 0x2B
XL_PRINTHEADERS = 0x2A
XL_RK = 0x27e
XL_ROW = 0x208
XL_ROW_B2 = 0x08
XL_RSTRING = 0xd6
XL_SCL = 0x00A0
XL_SHEETHDR = 0x8F # BIFF4W only
XL_SHEETPR = 0x81
XL_SHEETSOFFSET = 0x8E # BIFF4W only
XL_SHRFMLA = 0x04bc
XL_SST = 0xfc
XL_STANDARDWIDTH = 0x99
XL_STRING = 0x207
XL_STRING_B2 = 0x7
XL_STYLE = 0x293
XL_SUPBOOK = 0x1AE # aka EXTERNALBOOK in OOo docs
XL_TABLEOP = 0x236
XL_TABLEOP2 = 0x37
XL_TABLEOP_B2 = 0x36
XL_TXO = 0x1b6
XL_UNCALCED = 0x5e
XL_UNKNOWN = 0xffff
XL_VERTICALPAGEBREAKS = 0x1a
XL_WINDOW2 = 0x023E
XL_WINDOW2_B2 = 0x003E
XL_WRITEACCESS = 0x5C
XL_WSBOOL = XL_SHEETPR
XL_XF = 0xe0
XL_XF2 = 0x0043 # BIFF2 version of XF record
XL_XF3 = 0x0243 # BIFF3 version of XF record
XL_XF4 = 0x0443 # BIFF4 version of XF record
boflen = {0x0809: 8, 0x0409: 6, 0x0209: 6, 0x0009: 4}
bofcodes = (0x0809, 0x0409, 0x0209, 0x0009)
XL_FORMULA_OPCODES = (0x0006, 0x0406, 0x0206)
_cell_opcode_list = [
XL_BOOLERR,
XL_FORMULA,
XL_FORMULA3,
XL_FORMULA4,
XL_LABEL,
XL_LABELSST,
XL_MULRK,
XL_NUMBER,
XL_RK,
XL_RSTRING,
]
_cell_opcode_dict = {}
for _cell_opcode in _cell_opcode_list:
_cell_opcode_dict[_cell_opcode] = 1
def is_cell_opcode(c):
return c in _cell_opcode_dict
def upkbits(tgt_obj, src, manifest, local_setattr=setattr):
for n, mask, attr in manifest:
local_setattr(tgt_obj, attr, (src & mask) >> n)
def upkbitsL(tgt_obj, src, manifest, local_setattr=setattr, local_int=int):
for n, mask, attr in manifest:
local_setattr(tgt_obj, attr, local_int((src & mask) >> n))
def unpack_string(data, pos, encoding, lenlen=1):
nchars = unpack('<' + 'BH'[lenlen-1], data[pos:pos+lenlen])[0]
pos += lenlen
return unicode(data[pos:pos+nchars], encoding)
def unpack_string_update_pos(data, pos, encoding, lenlen=1, known_len=None):
if known_len is not None:
# On a NAME record, the length byte is detached from the front of the string.
nchars = known_len
else:
nchars = unpack('<' + 'BH'[lenlen-1], data[pos:pos+lenlen])[0]
pos += lenlen
newpos = pos + nchars
return (unicode(data[pos:newpos], encoding), newpos)
def unpack_unicode(data, pos, lenlen=2):
"Return unicode_strg"
nchars = unpack('<' + 'BH'[lenlen-1], data[pos:pos+lenlen])[0]
if not nchars:
# Ambiguous whether 0-length string should have an "options" byte.
# Avoid crash if missing.
return UNICODE_LITERAL("")
pos += lenlen
options = BYTES_ORD(data[pos])
pos += 1
# phonetic = options & 0x04
# richtext = options & 0x08
if options & 0x08:
# rt = unpack('<H', data[pos:pos+2])[0] # unused
pos += 2
if options & 0x04:
# sz = unpack('<i', data[pos:pos+4])[0] # unused
pos += 4
if options & 0x01:
# Uncompressed UTF-16-LE
rawstrg = data[pos:pos+2*nchars]
# if DEBUG: print "nchars=%d pos=%d rawstrg=%r" % (nchars, pos, rawstrg)
strg = unicode(rawstrg, 'utf_16_le')
# pos += 2*nchars
else:
# Note: this is COMPRESSED (not ASCII!) encoding!!!
# Merely returning the raw bytes would work OK 99.99% of the time
# if the local codepage was cp1252 -- however this would rapidly go pear-shaped
# for other codepages so we grit our Anglocentric teeth and return Unicode :-)
strg = unicode(data[pos:pos+nchars], "latin_1")
# pos += nchars
# if richtext:
# pos += 4 * rt
# if phonetic:
# pos += sz
# return (strg, pos)
return strg
def unpack_unicode_update_pos(data, pos, lenlen=2, known_len=None):
"Return (unicode_strg, updated value of pos)"
if known_len is not None:
# On a NAME record, the length byte is detached from the front of the string.
nchars = known_len
else:
nchars = unpack('<' + 'BH'[lenlen-1], data[pos:pos+lenlen])[0]
pos += lenlen
if not nchars and not data[pos:]:
# Zero-length string with no options byte
return (UNICODE_LITERAL(""), pos)
options = BYTES_ORD(data[pos])
pos += 1
phonetic = options & 0x04
richtext = options & 0x08
if richtext:
rt = unpack('<H', data[pos:pos+2])[0]
pos += 2
if phonetic:
sz = unpack('<i', data[pos:pos+4])[0]
pos += 4
if options & 0x01:
# Uncompressed UTF-16-LE
strg = unicode(data[pos:pos+2*nchars], 'utf_16_le')
pos += 2*nchars
else:
# Note: this is COMPRESSED (not ASCII!) encoding!!!
strg = unicode(data[pos:pos+nchars], "latin_1")
pos += nchars
if richtext:
pos += 4 * rt
if phonetic:
pos += sz
return (strg, pos)
def unpack_cell_range_address_list_update_pos(output_list, data, pos, biff_version, addr_size=6):
# output_list is updated in situ
assert addr_size in (6, 8)
# Used to assert size == 6 if not BIFF8, but pyWLWriter writes
# BIFF8-only MERGEDCELLS records in a BIFF5 file!
n, = unpack("<H", data[pos:pos+2])
pos += 2
if n:
if addr_size == 6:
fmt = "<HHBB"
else:
fmt = "<HHHH"
for _unused in xrange(n):
ra, rb, ca, cb = unpack(fmt, data[pos:pos+addr_size])
output_list.append((ra, rb+1, ca, cb+1))
pos += addr_size
return pos
_brecstrg = """\
0000 DIMENSIONS_B2
0001 BLANK_B2
0002 INTEGER_B2_ONLY
0003 NUMBER_B2
0004 LABEL_B2
0005 BOOLERR_B2
0006 FORMULA
0007 STRING_B2
0008 ROW_B2
0009 BOF_B2
000A EOF
000B INDEX_B2_ONLY
000C CALCCOUNT
000D CALCMODE
000E PRECISION
000F REFMODE
0010 DELTA
0011 ITERATION
0012 PROTECT
0013 PASSWORD
0014 HEADER
0015 FOOTER
0016 EXTERNCOUNT
0017 EXTERNSHEET
0018 NAME_B2,5+
0019 WINDOWPROTECT
001A VERTICALPAGEBREAKS
001B HORIZONTALPAGEBREAKS
001C NOTE
001D SELECTION
001E FORMAT_B2-3
001F BUILTINFMTCOUNT_B2
0020 COLUMNDEFAULT_B2_ONLY
0021 ARRAY_B2_ONLY
0022 DATEMODE
0023 EXTERNNAME
0024 COLWIDTH_B2_ONLY
0025 DEFAULTROWHEIGHT_B2_ONLY
0026 LEFTMARGIN
0027 RIGHTMARGIN
0028 TOPMARGIN
0029 BOTTOMMARGIN
002A PRINTHEADERS
002B PRINTGRIDLINES
002F FILEPASS
0031 FONT
0032 FONT2_B2_ONLY
0036 TABLEOP_B2
0037 TABLEOP2_B2
003C CONTINUE
003D WINDOW1
003E WINDOW2_B2
0040 BACKUP
0041 PANE
0042 CODEPAGE
0043 XF_B2
0044 IXFE_B2_ONLY
0045 EFONT_B2_ONLY
004D PLS
0051 DCONREF
0055 DEFCOLWIDTH
0056 BUILTINFMTCOUNT_B3-4
0059 XCT
005A CRN
005B FILESHARING
005C WRITEACCESS
005D OBJECT
005E UNCALCED
005F SAVERECALC
0063 OBJECTPROTECT
007D COLINFO
007E RK2_mythical_?
0080 GUTS
0081 WSBOOL
0082 GRIDSET
0083 HCENTER
0084 VCENTER
0085 BOUNDSHEET
0086 WRITEPROT
008C COUNTRY
008D HIDEOBJ
008E SHEETSOFFSET
008F SHEETHDR
0090 SORT
0092 PALETTE
0099 STANDARDWIDTH
009B FILTERMODE
009C FNGROUPCOUNT
009D AUTOFILTERINFO
009E AUTOFILTER
00A0 SCL
00A1 SETUP
00AB GCW
00BD MULRK
00BE MULBLANK
00C1 MMS
00D6 RSTRING
00D7 DBCELL
00DA BOOKBOOL
00DD SCENPROTECT
00E0 XF
00E1 INTERFACEHDR
00E2 INTERFACEEND
00E5 MERGEDCELLS
00E9 BITMAP
00EB MSO_DRAWING_GROUP
00EC MSO_DRAWING
00ED MSO_DRAWING_SELECTION
00EF PHONETIC
00FC SST
00FD LABELSST
00FF EXTSST
013D TABID
015F LABELRANGES
0160 USESELFS
0161 DSF
01AE SUPBOOK
01AF PROTECTIONREV4
01B0 CONDFMT
01B1 CF
01B2 DVAL
01B6 TXO
01B7 REFRESHALL
01B8 HLINK
01BC PASSWORDREV4
01BE DV
01C0 XL9FILE
01C1 RECALCID
0200 DIMENSIONS
0201 BLANK
0203 NUMBER
0204 LABEL
0205 BOOLERR
0206 FORMULA_B3
0207 STRING
0208 ROW
0209 BOF
020B INDEX_B3+
0218 NAME
0221 ARRAY
0223 EXTERNNAME_B3-4
0225 DEFAULTROWHEIGHT
0231 FONT_B3B4
0236 TABLEOP
023E WINDOW2
0243 XF_B3
027E RK
0293 STYLE
0406 FORMULA_B4
0409 BOF
041E FORMAT
0443 XF_B4
04BC SHRFMLA
0800 QUICKTIP
0809 BOF
0862 SHEETLAYOUT
0867 SHEETPROTECTION
0868 RANGEPROTECTION
"""
biff_rec_name_dict = {}
for _buff in _brecstrg.splitlines():
_numh, _name = _buff.split()
biff_rec_name_dict[int(_numh, 16)] = _name
del _buff, _name, _brecstrg
def hex_char_dump(strg, ofs, dlen, base=0, fout=sys.stdout, unnumbered=False):
endpos = min(ofs + dlen, len(strg))
pos = ofs
numbered = not unnumbered
num_prefix = ''
while pos < endpos:
endsub = min(pos + 16, endpos)
substrg = strg[pos:endsub]
lensub = endsub - pos
if lensub <= 0 or lensub != len(substrg):
fprintf(
sys.stdout,
'??? hex_char_dump: ofs=%d dlen=%d base=%d -> endpos=%d pos=%d endsub=%d substrg=%r\n',
ofs, dlen, base, endpos, pos, endsub, substrg)
break
hexd = ''.join("%02x " % BYTES_ORD(c) for c in substrg)
chard = ''
for c in substrg:
c = chr(BYTES_ORD(c))
if c == '\0':
c = '~'
elif not (' ' <= c <= '~'):
c = '?'
chard += c
if numbered:
num_prefix = "%5d: " % (base+pos-ofs)
fprintf(fout, "%s %-48s %s\n", num_prefix, hexd, chard)
pos = endsub
def biff_dump(mem, stream_offset, stream_len, base=0, fout=sys.stdout, unnumbered=False):
pos = stream_offset
stream_end = stream_offset + stream_len
adj = base - stream_offset
dummies = 0
numbered = not unnumbered
num_prefix = ''
while stream_end - pos >= 4:
rc, length = unpack('<HH', mem[pos:pos+4])
if rc == 0 and length == 0:
if mem[pos:] == b'\0' * (stream_end - pos):
dummies = stream_end - pos
savpos = pos
pos = stream_end
break
if dummies:
dummies += 4
else:
savpos = pos
dummies = 4
pos += 4
else:
if dummies:
if numbered:
num_prefix = "%5d: " % (adj + savpos)
fprintf(fout, "%s---- %d zero bytes skipped ----\n", num_prefix, dummies)
dummies = 0
recname = biff_rec_name_dict.get(rc, '<UNKNOWN>')
if numbered:
num_prefix = "%5d: " % (adj + pos)
fprintf(fout, "%s%04x %s len = %04x (%d)\n", num_prefix, rc, recname, length, length)
pos += 4
hex_char_dump(mem, pos, length, adj+pos, fout, unnumbered)
pos += length
if dummies:
if numbered:
num_prefix = "%5d: " % (adj + savpos)
fprintf(fout, "%s---- %d zero bytes skipped ----\n", num_prefix, dummies)
if pos < stream_end:
if numbered:
num_prefix = "%5d: " % (adj + pos)
fprintf(fout, "%s---- Misc bytes at end ----\n", num_prefix)
hex_char_dump(mem, pos, stream_end-pos, adj + pos, fout, unnumbered)
elif pos > stream_end:
fprintf(fout, "Last dumped record has length (%d) that is too large\n", length)
def biff_count_records(mem, stream_offset, stream_len, fout=sys.stdout):
pos = stream_offset
stream_end = stream_offset + stream_len
tally = {}
while stream_end - pos >= 4:
rc, length = unpack('<HH', mem[pos:pos+4])
if rc == 0 and length == 0:
if mem[pos:] == b'\0' * (stream_end - pos):
break
recname = "<Dummy (zero)>"
else:
recname = biff_rec_name_dict.get(rc, None)
if recname is None:
recname = "Unknown_0x%04X" % rc
if recname in tally:
tally[recname] += 1
else:
tally[recname] = 1
pos += length + 4
slist = sorted(tally.items())
for recname, count in slist:
print("%8d %s" % (count, recname), file=fout)
encoding_from_codepage = {
1200 : 'utf_16_le',
10000: 'mac_roman',
10006: 'mac_greek', # guess
10007: 'mac_cyrillic', # guess
10029: 'mac_latin2', # guess
10079: 'mac_iceland', # guess
10081: 'mac_turkish', # guess
32768: 'mac_roman',
32769: 'cp1252',
}
# some more guessing, for Indic scripts
# codepage 57000 range:
# 2 Devanagari [0]
# 3 Bengali [1]
# 4 Tamil [5]
# 5 Telegu [6]
# 6 Assamese [1] c.f. Bengali
# 7 Oriya [4]
# 8 Kannada [7]
# 9 Malayalam [8]
# 10 Gujarati [3]
# 11 Gurmukhi [2]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,485 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2005-2012 Stephen John Machin, Lingfo Pty Ltd
# This module is part of the xlrd package, which is released under a
# BSD-style licence.
# No part of the content of this file was derived from the works of
# David Giffin.
"""
Implements the minimal functionality required
to extract a "Workbook" or "Book" stream (as one big string)
from an OLE2 Compound Document file.
"""
from __future__ import print_function
import array
import sys
from struct import unpack
from .timemachine import *
#: Magic cookie that should appear in the first 8 bytes of the file.
SIGNATURE = b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1"
EOCSID = -2
FREESID = -1
SATSID = -3
MSATSID = -4
EVILSID = -5
class CompDocError(Exception):
pass
class DirNode(object):
def __init__(self, DID, dent, DEBUG=0, logfile=sys.stdout):
# dent is the 128-byte directory entry
self.DID = DID
self.logfile = logfile
(cbufsize, self.etype, self.colour, self.left_DID, self.right_DID,
self.root_DID) = \
unpack('<HBBiii', dent[64:80])
(self.first_SID, self.tot_size) = \
unpack('<ii', dent[116:124])
if cbufsize == 0:
self.name = UNICODE_LITERAL('')
else:
self.name = unicode(dent[0:cbufsize-2], 'utf_16_le') # omit the trailing U+0000
self.children = [] # filled in later
self.parent = -1 # indicates orphan; fixed up later
self.tsinfo = unpack('<IIII', dent[100:116])
if DEBUG:
self.dump(DEBUG)
def dump(self, DEBUG=1):
fprintf(
self.logfile,
"DID=%d name=%r etype=%d DIDs(left=%d right=%d root=%d parent=%d kids=%r) first_SID=%d tot_size=%d\n",
self.DID, self.name, self.etype, self.left_DID,
self.right_DID, self.root_DID, self.parent, self.children, self.first_SID, self.tot_size
)
if DEBUG == 2:
# cre_lo, cre_hi, mod_lo, mod_hi = tsinfo
print("timestamp info", self.tsinfo, file=self.logfile)
def _build_family_tree(dirlist, parent_DID, child_DID):
if child_DID < 0: return
_build_family_tree(dirlist, parent_DID, dirlist[child_DID].left_DID)
dirlist[parent_DID].children.append(child_DID)
dirlist[child_DID].parent = parent_DID
_build_family_tree(dirlist, parent_DID, dirlist[child_DID].right_DID)
if dirlist[child_DID].etype == 1: # storage
_build_family_tree(dirlist, child_DID, dirlist[child_DID].root_DID)
class CompDoc(object):
"""
Compound document handler.
:param mem:
The raw contents of the file, as a string, or as an :class:`mmap.mmap`
object. The only operation it needs to support is slicing.
"""
def __init__(self, mem, logfile=sys.stdout, DEBUG=0, ignore_workbook_corruption=False):
self.logfile = logfile
self.ignore_workbook_corruption = ignore_workbook_corruption
self.DEBUG = DEBUG
if mem[0:8] != SIGNATURE:
raise CompDocError('Not an OLE2 compound document')
if mem[28:30] != b'\xFE\xFF':
raise CompDocError('Expected "little-endian" marker, found %r' % mem[28:30])
revision, version = unpack('<HH', mem[24:28])
if DEBUG:
print("\nCompDoc format: version=0x%04x revision=0x%04x" % (version, revision), file=logfile)
self.mem = mem
ssz, sssz = unpack('<HH', mem[30:34])
if ssz > 20: # allows for 2**20 bytes i.e. 1MB
print("WARNING: sector size (2**%d) is preposterous; assuming 512 and continuing ..."
% ssz, file=logfile)
ssz = 9
if sssz > ssz:
print("WARNING: short stream sector size (2**%d) is preposterous; assuming 64 and continuing ..."
% sssz, file=logfile)
sssz = 6
self.sec_size = sec_size = 1 << ssz
self.short_sec_size = 1 << sssz
if self.sec_size != 512 or self.short_sec_size != 64:
print("@@@@ sec_size=%d short_sec_size=%d" % (self.sec_size, self.short_sec_size), file=logfile)
(
SAT_tot_secs, self.dir_first_sec_sid, _unused, self.min_size_std_stream,
SSAT_first_sec_sid, SSAT_tot_secs,
MSATX_first_sec_sid, MSATX_tot_secs,
) = unpack('<iiiiiiii', mem[44:76])
mem_data_len = len(mem) - 512
mem_data_secs, left_over = divmod(mem_data_len, sec_size)
if left_over:
#### raise CompDocError("Not a whole number of sectors")
mem_data_secs += 1
print("WARNING *** file size (%d) not 512 + multiple of sector size (%d)"
% (len(mem), sec_size), file=logfile)
self.mem_data_secs = mem_data_secs # use for checking later
self.mem_data_len = mem_data_len
seen = self.seen = array.array('B', [0]) * mem_data_secs
if DEBUG:
print('sec sizes', ssz, sssz, sec_size, self.short_sec_size, file=logfile)
print("mem data: %d bytes == %d sectors" % (mem_data_len, mem_data_secs), file=logfile)
print("SAT_tot_secs=%d, dir_first_sec_sid=%d, min_size_std_stream=%d"
% (SAT_tot_secs, self.dir_first_sec_sid, self.min_size_std_stream,), file=logfile)
print("SSAT_first_sec_sid=%d, SSAT_tot_secs=%d" % (SSAT_first_sec_sid, SSAT_tot_secs,), file=logfile)
print("MSATX_first_sec_sid=%d, MSATX_tot_secs=%d" % (MSATX_first_sec_sid, MSATX_tot_secs,), file=logfile)
nent = sec_size // 4 # number of SID entries in a sector
fmt = "<%di" % nent
trunc_warned = 0
#
# === build the MSAT ===
#
MSAT = list(unpack('<109i', mem[76:512]))
SAT_sectors_reqd = (mem_data_secs + nent - 1) // nent
expected_MSATX_sectors = max(0, (SAT_sectors_reqd - 109 + nent - 2) // (nent - 1))
actual_MSATX_sectors = 0
if MSATX_tot_secs == 0 and MSATX_first_sec_sid in (EOCSID, FREESID, 0):
# Strictly, if there is no MSAT extension, then MSATX_first_sec_sid
# should be set to EOCSID ... FREESID and 0 have been met in the wild.
pass # Presuming no extension
else:
sid = MSATX_first_sec_sid
while sid not in (EOCSID, FREESID, MSATSID):
# Above should be only EOCSID according to MS & OOo docs
# but Excel doesn't complain about FREESID. Zero is a valid
# sector number, not a sentinel.
if DEBUG > 1:
print('MSATX: sid=%d (0x%08X)' % (sid, sid), file=logfile)
if sid >= mem_data_secs:
msg = "MSAT extension: accessing sector %d but only %d in file" % (sid, mem_data_secs)
if DEBUG > 1:
print(msg, file=logfile)
break
raise CompDocError(msg)
elif sid < 0:
raise CompDocError("MSAT extension: invalid sector id: %d" % sid)
if seen[sid]:
raise CompDocError("MSAT corruption: seen[%d] == %d" % (sid, seen[sid]))
seen[sid] = 1
actual_MSATX_sectors += 1
if DEBUG and actual_MSATX_sectors > expected_MSATX_sectors:
print("[1]===>>>", mem_data_secs, nent, SAT_sectors_reqd, expected_MSATX_sectors, actual_MSATX_sectors, file=logfile)
offset = 512 + sec_size * sid
MSAT.extend(unpack(fmt, mem[offset:offset+sec_size]))
sid = MSAT.pop() # last sector id is sid of next sector in the chain
if DEBUG and actual_MSATX_sectors != expected_MSATX_sectors:
print("[2]===>>>", mem_data_secs, nent, SAT_sectors_reqd, expected_MSATX_sectors, actual_MSATX_sectors, file=logfile)
if DEBUG:
print("MSAT: len =", len(MSAT), file=logfile)
dump_list(MSAT, 10, logfile)
#
# === build the SAT ===
#
self.SAT = []
actual_SAT_sectors = 0
dump_again = 0
for msidx in xrange(len(MSAT)):
msid = MSAT[msidx]
if msid in (FREESID, EOCSID):
# Specification: the MSAT array may be padded with trailing FREESID entries.
# Toleration: a FREESID or EOCSID entry anywhere in the MSAT array will be ignored.
continue
if msid >= mem_data_secs:
if not trunc_warned:
print("WARNING *** File is truncated, or OLE2 MSAT is corrupt!!", file=logfile)
print("INFO: Trying to access sector %d but only %d available"
% (msid, mem_data_secs), file=logfile)
trunc_warned = 1
MSAT[msidx] = EVILSID
dump_again = 1
continue
elif msid < -2:
raise CompDocError("MSAT: invalid sector id: %d" % msid)
if seen[msid]:
raise CompDocError("MSAT extension corruption: seen[%d] == %d" % (msid, seen[msid]))
seen[msid] = 2
actual_SAT_sectors += 1
if DEBUG and actual_SAT_sectors > SAT_sectors_reqd:
print("[3]===>>>", mem_data_secs, nent, SAT_sectors_reqd, expected_MSATX_sectors, actual_MSATX_sectors, actual_SAT_sectors, msid, file=logfile)
offset = 512 + sec_size * msid
self.SAT.extend(unpack(fmt, mem[offset:offset+sec_size]))
if DEBUG:
print("SAT: len =", len(self.SAT), file=logfile)
dump_list(self.SAT, 10, logfile)
# print >> logfile, "SAT ",
# for i, s in enumerate(self.SAT):
# print >> logfile, "entry: %4d offset: %6d, next entry: %4d" % (i, 512 + sec_size * i, s)
# print >> logfile, "%d:%d " % (i, s),
print(file=logfile)
if DEBUG and dump_again:
print("MSAT: len =", len(MSAT), file=logfile)
dump_list(MSAT, 10, logfile)
for satx in xrange(mem_data_secs, len(self.SAT)):
self.SAT[satx] = EVILSID
print("SAT: len =", len(self.SAT), file=logfile)
dump_list(self.SAT, 10, logfile)
#
# === build the directory ===
#
dbytes = self._get_stream(
self.mem, 512, self.SAT, self.sec_size, self.dir_first_sec_sid,
name="directory", seen_id=3)
dirlist = []
did = -1
for pos in xrange(0, len(dbytes), 128):
did += 1
dirlist.append(DirNode(did, dbytes[pos:pos+128], 0, logfile))
self.dirlist = dirlist
_build_family_tree(dirlist, 0, dirlist[0].root_DID) # and stand well back ...
if DEBUG:
for d in dirlist:
d.dump(DEBUG)
#
# === get the SSCS ===
#
sscs_dir = self.dirlist[0]
assert sscs_dir.etype == 5 # root entry
if sscs_dir.first_SID < 0 or sscs_dir.tot_size == 0:
# Problem reported by Frank Hoffsuemmer: some software was
# writing -1 instead of -2 (EOCSID) for the first_SID
# when the SCCS was empty. Not having EOCSID caused assertion
# failure in _get_stream.
# Solution: avoid calling _get_stream in any case when the
# SCSS appears to be empty.
self.SSCS = ""
else:
self.SSCS = self._get_stream(
self.mem, 512, self.SAT, sec_size, sscs_dir.first_SID,
sscs_dir.tot_size, name="SSCS", seen_id=4)
# if DEBUG: print >> logfile, "SSCS", repr(self.SSCS)
#
# === build the SSAT ===
#
self.SSAT = []
if SSAT_tot_secs > 0 and sscs_dir.tot_size == 0:
print("WARNING *** OLE2 inconsistency: SSCS size is 0 but SSAT size is non-zero", file=logfile)
if sscs_dir.tot_size > 0:
sid = SSAT_first_sec_sid
nsecs = SSAT_tot_secs
while sid >= 0 and nsecs > 0:
if seen[sid]:
raise CompDocError("SSAT corruption: seen[%d] == %d" % (sid, seen[sid]))
seen[sid] = 5
nsecs -= 1
start_pos = 512 + sid * sec_size
news = list(unpack(fmt, mem[start_pos:start_pos+sec_size]))
self.SSAT.extend(news)
sid = self.SAT[sid]
if DEBUG: print("SSAT last sid %d; remaining sectors %d" % (sid, nsecs), file=logfile)
assert nsecs == 0 and sid == EOCSID
if DEBUG:
print("SSAT", file=logfile)
dump_list(self.SSAT, 10, logfile)
if DEBUG:
print("seen", file=logfile)
dump_list(seen, 20, logfile)
def _get_stream(self, mem, base, sat, sec_size, start_sid, size=None, name='', seen_id=None):
# print >> self.logfile, "_get_stream", base, sec_size, start_sid, size
sectors = []
s = start_sid
if size is None:
# nothing to check against
while s >= 0:
if seen_id is not None:
if self.seen[s]:
raise CompDocError("%s corruption: seen[%d] == %d" % (name, s, self.seen[s]))
self.seen[s] = seen_id
start_pos = base + s * sec_size
sectors.append(mem[start_pos:start_pos+sec_size])
try:
s = sat[s]
except IndexError:
raise CompDocError(
"OLE2 stream %r: sector allocation table invalid entry (%d)" %
(name, s)
)
assert s == EOCSID
else:
todo = size
while s >= 0:
if seen_id is not None:
if self.seen[s]:
raise CompDocError("%s corruption: seen[%d] == %d" % (name, s, self.seen[s]))
self.seen[s] = seen_id
start_pos = base + s * sec_size
grab = sec_size
if grab > todo:
grab = todo
todo -= grab
sectors.append(mem[start_pos:start_pos+grab])
try:
s = sat[s]
except IndexError:
raise CompDocError(
"OLE2 stream %r: sector allocation table invalid entry (%d)" %
(name, s)
)
assert s == EOCSID
if todo != 0:
fprintf(self.logfile,
"WARNING *** OLE2 stream %r: expected size %d, actual size %d\n",
name, size, size - todo)
return b''.join(sectors)
def _dir_search(self, path, storage_DID=0):
# Return matching DirNode instance, or None
head = path[0]
tail = path[1:]
dl = self.dirlist
for child in dl[storage_DID].children:
if dl[child].name.lower() == head.lower():
et = dl[child].etype
if et == 2:
return dl[child]
if et == 1:
if not tail:
raise CompDocError("Requested component is a 'storage'")
return self._dir_search(tail, child)
dl[child].dump(1)
raise CompDocError("Requested stream is not a 'user stream'")
return None
def get_named_stream(self, qname):
"""
Interrogate the compound document's directory; return the stream as a
string if found, otherwise return ``None``.
:param qname:
Name of the desired stream e.g. ``'Workbook'``.
Should be in Unicode or convertible thereto.
"""
d = self._dir_search(qname.split("/"))
if d is None:
return None
if d.tot_size >= self.min_size_std_stream:
return self._get_stream(
self.mem, 512, self.SAT, self.sec_size, d.first_SID,
d.tot_size, name=qname, seen_id=d.DID+6)
else:
return self._get_stream(
self.SSCS, 0, self.SSAT, self.short_sec_size, d.first_SID,
d.tot_size, name=qname + " (from SSCS)", seen_id=None)
def locate_named_stream(self, qname):
"""
Interrogate the compound document's directory.
If the named stream is not found, ``(None, 0, 0)`` will be returned.
If the named stream is found and is contiguous within the original
byte sequence (``mem``) used when the document was opened,
then ``(mem, offset_to_start_of_stream, length_of_stream)`` is returned.
Otherwise a new string is built from the fragments and
``(new_string, 0, length_of_stream)`` is returned.
:param qname:
Name of the desired stream e.g. ``'Workbook'``.
Should be in Unicode or convertible thereto.
"""
d = self._dir_search(qname.split("/"))
if d is None:
return (None, 0, 0)
if d.tot_size > self.mem_data_len:
raise CompDocError("%r stream length (%d bytes) > file data size (%d bytes)"
% (qname, d.tot_size, self.mem_data_len))
if d.tot_size >= self.min_size_std_stream:
result = self._locate_stream(
self.mem, 512, self.SAT, self.sec_size, d.first_SID,
d.tot_size, qname, d.DID+6)
if self.DEBUG:
print("\nseen", file=self.logfile)
dump_list(self.seen, 20, self.logfile)
return result
else:
return (
self._get_stream(
self.SSCS, 0, self.SSAT, self.short_sec_size, d.first_SID,
d.tot_size, qname + " (from SSCS)", None),
0,
d.tot_size,
)
def _locate_stream(self, mem, base, sat, sec_size, start_sid, expected_stream_size, qname, seen_id):
# print >> self.logfile, "_locate_stream", base, sec_size, start_sid, expected_stream_size
s = start_sid
if s < 0:
raise CompDocError("_locate_stream: start_sid (%d) is -ve" % start_sid)
p = -99 # dummy previous SID
start_pos = -9999
end_pos = -8888
slices = []
tot_found = 0
found_limit = (expected_stream_size + sec_size - 1) // sec_size
while s >= 0:
if self.seen[s]:
if not self.ignore_workbook_corruption:
print("_locate_stream(%s): seen" % qname, file=self.logfile); dump_list(self.seen, 20, self.logfile)
raise CompDocError("%s corruption: seen[%d] == %d" % (qname, s, self.seen[s]))
self.seen[s] = seen_id
tot_found += 1
if tot_found > found_limit:
# Note: expected size rounded up to higher sector
raise CompDocError(
"%s: size exceeds expected %d bytes; corrupt?"
% (qname, found_limit * sec_size)
)
if s == p+1:
# contiguous sectors
end_pos += sec_size
else:
# start new slice
if p >= 0:
# not first time
slices.append((start_pos, end_pos))
start_pos = base + s * sec_size
end_pos = start_pos + sec_size
p = s
s = sat[s]
assert s == EOCSID
assert tot_found == found_limit
# print >> self.logfile, "_locate_stream(%s): seen" % qname; dump_list(self.seen, 20, self.logfile)
if not slices:
# The stream is contiguous ... just what we like!
return (mem, start_pos, expected_stream_size)
slices.append((start_pos, end_pos))
# print >> self.logfile, "+++>>> %d fragments" % len(slices)
return (b''.join(mem[start_pos:end_pos] for start_pos, end_pos in slices), 0, expected_stream_size)
# ==========================================================================================
def x_dump_line(alist, stride, f, dpos, equal=0):
print("%5d%s" % (dpos, " ="[equal]), end=' ', file=f)
for value in alist[dpos:dpos + stride]:
print(str(value), end=' ', file=f)
print(file=f)
def dump_list(alist, stride, f=sys.stdout):
def _dump_line(dpos, equal=0):
print("%5d%s" % (dpos, " ="[equal]), end=' ', file=f)
for value in alist[dpos:dpos + stride]:
print(str(value), end=' ', file=f)
print(file=f)
pos = None
oldpos = None
for pos in xrange(0, len(alist), stride):
if oldpos is None:
_dump_line(pos)
oldpos = pos
elif alist[pos:pos+stride] != alist[oldpos:oldpos+stride]:
if pos - oldpos > stride:
_dump_line(pos - stride, equal=1)
_dump_line(pos)
oldpos = pos
if oldpos is not None and pos is not None and pos != oldpos:
_dump_line(pos, equal=1)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1 @@
__version__ = __VERSION__ = "2.0.2"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,53 @@
##
# <p>Copyright (c) 2006-2012 Stephen John Machin, Lingfo Pty Ltd</p>
# <p>This module is part of the xlrd package, which is released under a BSD-style licence.</p>
##
# timemachine.py -- adaptation for single codebase.
# Currently supported: 2.6 to 2.7, 3.2+
# usage: from timemachine import *
from __future__ import print_function
import sys
python_version = sys.version_info[:2] # e.g. version 2.6 -> (2, 6)
if python_version >= (3, 0):
# Python 3
BYTES_LITERAL = lambda x: x.encode('latin1')
UNICODE_LITERAL = lambda x: x
BYTES_ORD = lambda byte: byte
from io import BytesIO as BYTES_IO
def fprintf(f, fmt, *vargs):
fmt = fmt.replace("%r", "%a")
if fmt.endswith('\n'):
print(fmt[:-1] % vargs, file=f)
else:
print(fmt % vargs, end=' ', file=f)
EXCEL_TEXT_TYPES = (str, bytes, bytearray) # xlwt: isinstance(obj, EXCEL_TEXT_TYPES)
REPR = ascii
xrange = range
unicode = lambda b, enc: b.decode(enc)
ensure_unicode = lambda s: s
unichr = chr
else:
# Python 2
BYTES_LITERAL = lambda x: x
UNICODE_LITERAL = lambda x: x.decode('latin1')
BYTES_ORD = ord
from cStringIO import StringIO as BYTES_IO
def fprintf(f, fmt, *vargs):
if fmt.endswith('\n'):
print(fmt[:-1] % vargs, file=f)
else:
print(fmt % vargs, end=' ', file=f)
try:
EXCEL_TEXT_TYPES = basestring # xlwt: isinstance(obj, EXCEL_TEXT_TYPES)
except NameError:
EXCEL_TEXT_TYPES = (str, unicode)
REPR = repr
xrange = xrange
# following used only to overcome 2.x ElementTree gimmick which
# returns text as `str` if it's ascii, otherwise `unicode`
ensure_unicode = unicode # used only in xlsx.py
@@ -0,0 +1,248 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2005-2008 Stephen John Machin, Lingfo Pty Ltd
# This module is part of the xlrd package, which is released under a
# BSD-style licence.
# No part of the content of this file was derived from the works of David Giffin.
"""
Tools for working with dates and times in Excel files.
The conversion from ``days`` to ``(year, month, day)`` starts with
an integral "julian day number" aka JDN.
FWIW:
- JDN 0 corresponds to noon on Monday November 24 in Gregorian year -4713.
More importantly:
- Noon on Gregorian 1900-03-01 (day 61 in the 1900-based system) is JDN 2415080.0
- Noon on Gregorian 1904-01-02 (day 1 in the 1904-based system) is JDN 2416482.0
"""
import datetime
_JDN_delta = (2415080 - 61, 2416482 - 1)
assert _JDN_delta[1] - _JDN_delta[0] == 1462
# Pre-calculate the datetime epochs for efficiency.
epoch_1904 = datetime.datetime(1904, 1, 1)
epoch_1900 = datetime.datetime(1899, 12, 31)
epoch_1900_minus_1 = datetime.datetime(1899, 12, 30)
# This is equivalent to 10000-01-01:
_XLDAYS_TOO_LARGE = (2958466, 2958466 - 1462)
class XLDateError(ValueError):
"A base class for all datetime-related errors."
class XLDateNegative(XLDateError):
"``xldate < 0.00``"
class XLDateAmbiguous(XLDateError):
"The 1900 leap-year problem ``(datemode == 0 and 1.0 <= xldate < 61.0)``"
class XLDateTooLarge(XLDateError):
"Gregorian year 10000 or later"
class XLDateBadDatemode(XLDateError):
"``datemode`` arg is neither 0 nor 1"
class XLDateBadTuple(XLDateError):
pass
def xldate_as_tuple(xldate, datemode):
"""
Convert an Excel number (presumed to represent a date, a datetime or a time) into
a tuple suitable for feeding to datetime or mx.DateTime constructors.
:param xldate: The Excel number
:param datemode: 0: 1900-based, 1: 1904-based.
:raises xlrd.xldate.XLDateNegative:
:raises xlrd.xldate.XLDateAmbiguous:
:raises xlrd.xldate.XLDateTooLarge:
:raises xlrd.xldate.XLDateBadDatemode:
:raises xlrd.xldate.XLDateError:
:returns: Gregorian ``(year, month, day, hour, minute, nearest_second)``.
.. warning::
When using this function to interpret the contents of a workbook, you
should pass in the :attr:`~xlrd.book.Book.datemode`
attribute of that workbook. Whether the workbook has ever been anywhere
near a Macintosh is irrelevant.
.. admonition:: Special case
If ``0.0 <= xldate < 1.0``, it is assumed to represent a time;
``(0, 0, 0, hour, minute, second)`` will be returned.
.. note::
``1904-01-01`` is not regarded as a valid date in the ``datemode==1``
system; its "serial number" is zero.
"""
if datemode not in (0, 1):
raise XLDateBadDatemode(datemode)
if xldate == 0.00:
return (0, 0, 0, 0, 0, 0)
if xldate < 0.00:
raise XLDateNegative(xldate)
xldays = int(xldate)
frac = xldate - xldays
seconds = int(round(frac * 86400.0))
assert 0 <= seconds <= 86400
if seconds == 86400:
hour = minute = second = 0
xldays += 1
else:
# second = seconds % 60; minutes = seconds // 60
minutes, second = divmod(seconds, 60)
# minute = minutes % 60; hour = minutes // 60
hour, minute = divmod(minutes, 60)
if xldays >= _XLDAYS_TOO_LARGE[datemode]:
raise XLDateTooLarge(xldate)
if xldays == 0:
return (0, 0, 0, hour, minute, second)
if xldays < 61 and datemode == 0:
raise XLDateAmbiguous(xldate)
jdn = xldays + _JDN_delta[datemode]
yreg = ((((jdn * 4 + 274277) // 146097) * 3 // 4) + jdn + 1363) * 4 + 3
mp = ((yreg % 1461) // 4) * 535 + 333
d = ((mp % 16384) // 535) + 1
# mp /= 16384
mp >>= 14
if mp >= 10:
return ((yreg // 1461) - 4715, mp - 9, d, hour, minute, second)
else:
return ((yreg // 1461) - 4716, mp + 3, d, hour, minute, second)
def xldate_as_datetime(xldate, datemode):
"""
Convert an Excel date/time number into a :class:`datetime.datetime` object.
:param xldate: The Excel number
:param datemode: 0: 1900-based, 1: 1904-based.
:returns: A :class:`datetime.datetime` object.
"""
# Set the epoch based on the 1900/1904 datemode.
if datemode:
epoch = epoch_1904
else:
if xldate < 60:
epoch = epoch_1900
else:
# Workaround Excel 1900 leap year bug by adjusting the epoch.
epoch = epoch_1900_minus_1
# The integer part of the Excel date stores the number of days since
# the epoch and the fractional part stores the percentage of the day.
days = int(xldate)
fraction = xldate - days
# Get the the integer and decimal seconds in Excel's millisecond resolution.
seconds = int(round(fraction * 86400000.0))
seconds, milliseconds = divmod(seconds, 1000)
return epoch + datetime.timedelta(days, seconds, 0, milliseconds)
# === conversions from date/time to xl numbers
def _leap(y):
if y % 4: return 0
if y % 100: return 1
if y % 400: return 0
return 1
_days_in_month = (None, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31)
def xldate_from_date_tuple(date_tuple, datemode):
"""
Convert a date tuple (year, month, day) to an Excel date.
:param year: Gregorian year.
:param month: ``1 <= month <= 12``
:param day: ``1 <= day <= last day of that (year, month)``
:param datemode: 0: 1900-based, 1: 1904-based.
:raises xlrd.xldate.XLDateAmbiguous:
:raises xlrd.xldate.XLDateBadDatemode:
:raises xlrd.xldate.XLDateBadTuple:
``(year, month, day)`` is too early/late or has invalid component(s)
:raises xlrd.xldate.XLDateError:
"""
year, month, day = date_tuple
if datemode not in (0, 1):
raise XLDateBadDatemode(datemode)
if year == 0 and month == 0 and day == 0:
return 0.00
if not (1900 <= year <= 9999):
raise XLDateBadTuple("Invalid year: %r" % ((year, month, day),))
if not (1 <= month <= 12):
raise XLDateBadTuple("Invalid month: %r" % ((year, month, day),))
if (day < 1 or
(day > _days_in_month[month] and not(day == 29 and month == 2 and _leap(year)))):
raise XLDateBadTuple("Invalid day: %r" % ((year, month, day),))
Yp = year + 4716
M = month
if M <= 2:
Yp = Yp - 1
Mp = M + 9
else:
Mp = M - 3
jdn = (1461 * Yp // 4) + ((979 * Mp + 16) // 32) + \
day - 1364 - (((Yp + 184) // 100) * 3 // 4)
xldays = jdn - _JDN_delta[datemode]
if xldays <= 0:
raise XLDateBadTuple("Invalid (year, month, day): %r" % ((year, month, day),))
if xldays < 61 and datemode == 0:
raise XLDateAmbiguous("Before 1900-03-01: %r" % ((year, month, day),))
return float(xldays)
def xldate_from_time_tuple(time_tuple):
"""
Convert a time tuple ``(hour, minute, second)`` to an Excel "date" value
(fraction of a day).
:param hour: ``0 <= hour < 24``
:param minute: ``0 <= minute < 60``
:param second: ``0 <= second < 60``
:raises xlrd.xldate.XLDateBadTuple: Out-of-range hour, minute, or second
"""
hour, minute, second = time_tuple
if 0 <= hour < 24 and 0 <= minute < 60 and 0 <= second < 60:
return ((second / 60.0 + minute) / 60.0 + hour) / 24.0
raise XLDateBadTuple("Invalid (hour, minute, second): %r" % ((hour, minute, second),))
def xldate_from_datetime_tuple(datetime_tuple, datemode):
"""
Convert a datetime tuple ``(year, month, day, hour, minute, second)`` to an
Excel date value.
For more details, refer to other xldate_from_*_tuple functions.
:param datetime_tuple: ``(year, month, day, hour, minute, second)``
:param datemode: 0: 1900-based, 1: 1904-based.
"""
return (
xldate_from_date_tuple(datetime_tuple[:3], datemode) +
xldate_from_time_tuple(datetime_tuple[3:])
)